missing options crash test

db_stress_tool/db_stress_common.h

@@ -20,6 +20,7 @@
 // NOTE that if FLAGS_test_batches_snapshots is set, the test will have
 // different behavior. See comment of the flag for details.
 
+#include <gflags/gflags_declare.h>
 #ifdef GFLAGS
 #pragma once
 #include <fcntl.h>
@@ -352,6 +353,41 @@ DECLARE_uint64(initial_auto_readahead_size);
 DECLARE_uint64(max_auto_readahead_size);
 DECLARE_uint64(num_file_reads_for_auto_readahead);
 DECLARE_bool(auto_readahead_size);
+DECLARE_bool(allow_fallocate);
+DECLARE_int32(table_cache_numshardbits);
+DECLARE_bool(is_fd_close_on_exec);
+DECLARE_bool(enable_write_thread_adaptive_yield);
+DECLARE_uint32(lowest_used_cache_tier);
+DECLARE_uint32(log_readahead_size);
+DECLARE_uint32(bgerror_resume_retry_interval);
+DECLARE_uint64(delete_obsolete_files_period_micros);
+DECLARE_uint32(max_log_file_size);
+DECLARE_uint32(log_file_time_to_roll);
+DECLARE_bool(use_adaptive_mutex);
+DECLARE_bool(advise_random_on_open);
+DECLARE_uint32(WAL_ttl_seconds);
+DECLARE_uint32(WAL_size_limit_MB);
+DECLARE_bool(strict_bytes_per_sync);
+DECLARE_bool(avoid_flush_during_shutdown);
+DECLARE_bool(fill_cache);
+DECLARE_bool(optimize_multiget_for_io);
+DECLARE_bool(low_pri);
+DECLARE_bool(memtable_insert_hint_per_batch);
+DECLARE_bool(dump_malloc_stats);
+DECLARE_bool(persist_stats_to_disk);
+DECLARE_uint32(stats_history_buffer_size);
+DECLARE_bool(skip_stats_update_on_db_open);
+DECLARE_bool(optimize_filters_for_hits);
+DECLARE_uint32(sample_for_compression);
+DECLARE_bool(report_bg_io_stats);
+DECLARE_bool(cache_index_and_filter_blocks_with_high_priority);
+DECLARE_bool(no_block_cache);
+DECLARE_bool(use_delta_encoding);
+DECLARE_bool(verify_compression);
+DECLARE_uint32(read_amp_bytes_per_bit);
+DECLARE_bool(enable_index_compression);
+DECLARE_bool(block_align);
+DECLARE_uint32(index_shortening);
 
 constexpr long KB = 1024;
 constexpr int kRandomValueMaxFactor = 3;

db_stress_tool/db_stress_gflags.cc

@@ -7,6 +7,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
+#include <gflags/gflags.h>
+
+#include "rocksdb/options.h"
 #ifdef GFLAGS
 #include "db_stress_tool/db_stress_common.h"
 
@@ -402,7 +405,8 @@ DEFINE_double(experimental_mempurge_threshold, 0.0,
               "Maximum estimated useful payload that triggers a "
               "mempurge process to collect memtable garbage bytes.");
 
-DEFINE_bool(enable_write_thread_adaptive_yield, true,
+DEFINE_bool(enable_write_thread_adaptive_yield,
+            ROCKSDB_NAMESPACE::Options().enable_write_thread_adaptive_yield,
             "Use a yielding spin loop for brief writer thread waits.");
 
 // Options for StackableDB-based BlobDB
@@ -1144,4 +1148,136 @@ DEFINE_uint32(bottommost_file_compaction_delay, 0,
 DEFINE_bool(auto_readahead_size, false,
             "Does auto tuning of readahead_size when enabled during scans.");
 
+DEFINE_bool(allow_fallocate, ROCKSDB_NAMESPACE::Options().allow_fallocate,
+            "Options.allow_fallocate");
+
+DEFINE_int32(table_cache_numshardbits,
+             ROCKSDB_NAMESPACE::Options().table_cache_numshardbits,
+             "Options.table_cache_numshardbits");
+
+DEFINE_bool(is_fd_close_on_exec,
+            ROCKSDB_NAMESPACE::Options().is_fd_close_on_exec,
+            "Options.is_fd_close_on_exec");
+
+DEFINE_uint32(
+    lowest_used_cache_tier,
+    static_cast<uint32_t>(ROCKSDB_NAMESPACE::Options().lowest_used_cache_tier),
+    "Options.lowest_used_cache_tier");
+
+DEFINE_uint32(log_readahead_size,
+              ROCKSDB_NAMESPACE::Options().log_readahead_size,
+              "Options.log_readahead_size");
+
+DEFINE_uint32(bgerror_resume_retry_interval,
+              ROCKSDB_NAMESPACE::Options().bgerror_resume_retry_interval,
+              "Options.bgerror_resume_retry_interval");
+
+DEFINE_uint64(delete_obsolete_files_period_micros,
+              ROCKSDB_NAMESPACE::Options().delete_obsolete_files_period_micros,
+              "Options.delete_obsolete_files_period_micros");
+
+DEFINE_uint32(max_log_file_size, ROCKSDB_NAMESPACE::Options().max_log_file_size,
+              "Options.max_log_file_sizes");
+
+DEFINE_uint32(log_file_time_to_roll,
+              ROCKSDB_NAMESPACE::Options().log_file_time_to_roll,
+              "Options.log_file_time_to_roll");
+
+DEFINE_bool(use_adaptive_mutex, ROCKSDB_NAMESPACE::Options().use_adaptive_mutex,
+            "Options.use_adaptive_mutex");
+
+DEFINE_bool(advise_random_on_open,
+            ROCKSDB_NAMESPACE::Options().advise_random_on_open,
+            "Options.advise_random_on_open");
+
+DEFINE_uint32(WAL_ttl_seconds, ROCKSDB_NAMESPACE::Options().WAL_ttl_seconds,
+              "Options.WAL_ttl_seconds");
+
+DEFINE_uint32(WAL_size_limit_MB, ROCKSDB_NAMESPACE::Options().WAL_size_limit_MB,
+              "Options.WAL_size_limit_MB");
+
+DEFINE_bool(strict_bytes_per_sync,
+            ROCKSDB_NAMESPACE::Options().strict_bytes_per_sync,
+            "Options.strict_bytes_per_sync");
+
+DEFINE_bool(avoid_flush_during_shutdown,
+            ROCKSDB_NAMESPACE::Options().avoid_flush_during_shutdown,
+            "Options.avoid_flush_during_shutdown");
+
+DEFINE_bool(fill_cache, ROCKSDB_NAMESPACE::ReadOptions().fill_cache,
+            "ReadOptions.fill_cache");
+
+DEFINE_bool(optimize_multiget_for_io,
+            ROCKSDB_NAMESPACE::ReadOptions().optimize_multiget_for_io,
+            "ReadOptions.optimize_multiget_for_io");
+
+DEFINE_bool(low_pri, ROCKSDB_NAMESPACE::WriteOptions().low_pri,
+            "WriteOptions.low_pri");
+
+DEFINE_bool(memtable_insert_hint_per_batch,
+            ROCKSDB_NAMESPACE::WriteOptions().memtable_insert_hint_per_batch,
+            "WriteOptions.memtable_insert_hint_per_batch");
+
+DEFINE_bool(dump_malloc_stats, ROCKSDB_NAMESPACE::Options().dump_malloc_stats,
+            "Options.dump_malloc_stats");
+
+DEFINE_bool(persist_stats_to_disk,
+            ROCKSDB_NAMESPACE::Options().persist_stats_to_disk,
+            "Options.persist_stats_to_disk");
+
+DEFINE_uint32(stats_history_buffer_size,
+              ROCKSDB_NAMESPACE::Options().stats_history_buffer_size,
+              "Options.stats_history_buffer_size");
+
+DEFINE_bool(skip_stats_update_on_db_open,
+            ROCKSDB_NAMESPACE::Options().skip_stats_update_on_db_open,
+            "Options.skip_stats_update_on_db_open");
+
+DEFINE_bool(optimize_filters_for_hits,
+            ROCKSDB_NAMESPACE::Options().optimize_filters_for_hits,
+            "Options.optimize_filters_for_hits");
+
+DEFINE_uint32(sample_for_compression,
+              ROCKSDB_NAMESPACE::Options().sample_for_compression,
+              "Options.sample_for_compression");
+
+DEFINE_bool(report_bg_io_stats, ROCKSDB_NAMESPACE::Options().report_bg_io_stats,
+            "Options.report_bg_io_stats");
+
+DEFINE_bool(
+    cache_index_and_filter_blocks_with_high_priority,
+    ROCKSDB_NAMESPACE::BlockBasedTableOptions()
+        .cache_index_and_filter_blocks_with_high_priority,
+    "BlockBasedTableOptions.cache_index_and_filter_blocks_with_high_priority");
+
+DEFINE_bool(no_block_cache,
+            ROCKSDB_NAMESPACE::BlockBasedTableOptions().no_block_cache,
+            "BlockBasedTableOptions.no_block_cache");
+
+DEFINE_bool(use_delta_encoding,
+            ROCKSDB_NAMESPACE::BlockBasedTableOptions().use_delta_encoding,
+            "BlockBasedTableOptions.use_delta_encoding");
+
+DEFINE_bool(verify_compression,
+            ROCKSDB_NAMESPACE::BlockBasedTableOptions().verify_compression,
+            "BlockBasedTableOptions.verify_compression");
+
+DEFINE_uint32(
+    read_amp_bytes_per_bit,
+    ROCKSDB_NAMESPACE::BlockBasedTableOptions().read_amp_bytes_per_bit,
+    "Options.read_amp_bytes_per_bit");
+
+DEFINE_bool(
+    enable_index_compression,
+    ROCKSDB_NAMESPACE::BlockBasedTableOptions().enable_index_compression,
+    "BlockBasedTableOptions.enable_index_compression");
+
+DEFINE_bool(block_align,
+            ROCKSDB_NAMESPACE::BlockBasedTableOptions().block_align,
+            "BlockBasedTableOptions.block_align");
+
+DEFINE_uint32(index_shortening,
+              static_cast<uint32_t>(
+                  ROCKSDB_NAMESPACE::BlockBasedTableOptions().index_shortening),
+              "BlockBasedTableOptions.index_shortening");
 #endif  // GFLAGS

db_stress_tool/db_stress_test_base.cc

@@ -830,10 +830,16 @@ void StressTest::OperateDb(ThreadState* thread) {
   read_opts.adaptive_readahead = FLAGS_adaptive_readahead;
   read_opts.readahead_size = FLAGS_readahead_size;
   read_opts.auto_readahead_size = FLAGS_auto_readahead_size;
+  read_opts.fill_cache = FLAGS_fill_cache;
+  read_opts.optimize_multiget_for_io = FLAGS_optimize_multiget_for_io;
   WriteOptions write_opts;
   if (FLAGS_rate_limit_auto_wal_flush) {
     write_opts.rate_limiter_priority = Env::IO_USER;
   }
+  // write_opts.low_pri = FLAGS_low_pri;
+  write_opts.memtable_insert_hint_per_batch =
+      FLAGS_memtable_insert_hint_per_batch;
+
   auto shared = thread->shared;
   char value[100];
   std::string from_db;
@@ -1264,6 +1270,10 @@ Status StressTest::TestIterate(ThreadState* thread,
   Slice read_ts_slice;
   MaybeUseOlderTimestampForRangeScan(thread, read_ts_str, read_ts_slice, ro);
 
+  // ro.tailing = thread->rand.OneIn(4);
+  ro.pin_data = thread->rand.OneIn(2);
+  ro.background_purge_on_iterator_cleanup = thread->rand.OneIn(2);
+
   bool expect_total_order = false;
   if (thread->rand.OneIn(16)) {
     // When prefix extractor is used, it's useful to cover total order seek.
@@ -2396,6 +2406,8 @@ void StressTest::TestCompactRange(ThreadState* thread, int64_t rand_key,
   CompactRangeOptions cro;
   cro.exclusive_manual_compaction = static_cast<bool>(thread->rand.Next() % 2);
   cro.change_level = static_cast<bool>(thread->rand.Next() % 2);
+  cro.target_level =
+      thread->rand.OneIn(2) ? thread->rand.Next() % options_.num_levels : -1;
   std::vector<BottommostLevelCompaction> bottom_level_styles = {
       BottommostLevelCompaction::kSkip,
       BottommostLevelCompaction::kIfHaveCompactionFilter,
@@ -3286,6 +3298,17 @@ void InitializeOptionsFromFlags(
   block_based_options.max_auto_readahead_size = FLAGS_max_auto_readahead_size;
   block_based_options.num_file_reads_for_auto_readahead =
       FLAGS_num_file_reads_for_auto_readahead;
+  block_based_options.cache_index_and_filter_blocks_with_high_priority =
+      FLAGS_cache_index_and_filter_blocks_with_high_priority;
+  // block_based_options.no_block_cache = FLAGS_no_block_cache;
+  block_based_options.use_delta_encoding = FLAGS_use_delta_encoding;
+  block_based_options.verify_compression = FLAGS_verify_compression;
+  block_based_options.read_amp_bytes_per_bit = FLAGS_read_amp_bytes_per_bit;
+  block_based_options.enable_index_compression = FLAGS_enable_index_compression;
+  // block_based_options.block_align = FLAGS_block_align;
+  block_based_options.index_shortening =
+      static_cast<BlockBasedTableOptions::IndexShorteningMode>(
+          FLAGS_index_shortening);
   options.table_factory.reset(NewBlockBasedTableFactory(block_based_options));
   options.db_write_buffer_size = FLAGS_db_write_buffer_size;
   options.write_buffer_size = FLAGS_write_buffer_size;
@@ -3486,6 +3509,34 @@ void InitializeOptionsFromFlags(
 
   options.bottommost_file_compaction_delay =
       FLAGS_bottommost_file_compaction_delay;
+
+  options.allow_fallocate = FLAGS_allow_fallocate;
+  options.table_cache_numshardbits = FLAGS_table_cache_numshardbits;
+  options.is_fd_close_on_exec = FLAGS_is_fd_close_on_exec;
+  options.enable_write_thread_adaptive_yield =
+      FLAGS_enable_write_thread_adaptive_yield;
+  // options.lowest_used_cache_tier =
+  //     static_cast<CacheTier>(FLAGS_lowest_used_cache_tier);
+  options.log_readahead_size = FLAGS_log_readahead_size;
+  options.bgerror_resume_retry_interval = FLAGS_bgerror_resume_retry_interval;
+  options.delete_obsolete_files_period_micros =
+      FLAGS_delete_obsolete_files_period_micros;
+  options.max_log_file_size = FLAGS_max_log_file_size;
+  options.log_file_time_to_roll = FLAGS_log_file_time_to_roll;
+  options.use_adaptive_mutex = FLAGS_use_adaptive_mutex;
+  options.advise_random_on_open = FLAGS_advise_random_on_open;
+  options.WAL_ttl_seconds = FLAGS_WAL_ttl_seconds;
+  options.WAL_size_limit_MB = FLAGS_WAL_size_limit_MB;
+  options.wal_bytes_per_sync = FLAGS_wal_bytes_per_sync;
+  options.strict_bytes_per_sync = FLAGS_strict_bytes_per_sync;
+  options.avoid_flush_during_shutdown = FLAGS_avoid_flush_during_shutdown;
+  options.dump_malloc_stats = FLAGS_dump_malloc_stats;
+  // options.persist_stats_to_disk = FLAGS_persist_stats_to_disk;
+  options.stats_history_buffer_size = FLAGS_stats_history_buffer_size;
+  options.skip_stats_update_on_db_open = FLAGS_skip_stats_update_on_db_open;
+  options.optimize_filters_for_hits = FLAGS_optimize_filters_for_hits;
+  options.sample_for_compression = FLAGS_sample_for_compression;
+  options.report_bg_io_stats = FLAGS_report_bg_io_stats;
 }
 
 void InitializeOptionsGeneral(

db_stress_tool/no_batched_ops_stress.cc

@@ -1594,8 +1594,14 @@ class NonBatchedOpsStressTest : public StressTest {
       s = sst_file_writer.Finish();
     }
     if (s.ok()) {
+      IngestExternalFileOptions ingest_options;
+      ingest_options.move_files = thread->rand.OneInOpt(2);
+      // ingest_options.ingest_behind = thread->rand.OneInOpt(2);
+      ingest_options.verify_checksums_before_ingest = thread->rand.OneInOpt(2);
+      ingest_options.verify_checksums_readahead_size =
+          thread->rand.OneInOpt(2) ? 1024 * 1024 : 0;
       s = db_->IngestExternalFile(column_families_[column_family],
-                                  {sst_filename}, IngestExternalFileOptions());
+                                  {sst_filename}, ingest_options);
     }
     if (!s.ok()) {
       for (PendingExpectedValue& pending_expected_value :

missing_options_crash_test/aoptionsh.out

@@ -0,0 +1,70 @@
+temperature
+age
+
+max_table_files_size
+allow_compaction
+age_for_warm
+file_temperature_age_thresholds
+
+strict_max_successive_merges
+max_write_buffer_number
+min_write_buffer_number_to_merge
+max_write_buffer_number_to_maintain
+max_write_buffer_size_to_maintain
+inplace_update_support
+inplace_update_num_locks
+experimental_mempurge_threshold
+inplace_callback
+memtable_prefix_bloom_size_ratio
+memtable_whole_key_filtering
+memtable_huge_page_size
+memtable_insert_with_hint_prefix_extractor
+bloom_locality
+arena_block_size
+compression_per_level
+num_levels
+level0_slowdown_writes_trigger
+level0_stop_writes_trigger
+target_file_size_base
+target_file_size_multiplier
+level_compaction_dynamic_level_bytes
+max_bytes_for_level_multiplier
+max_bytes_for_level_multiplier_additional
+max_compaction_bytes
+soft_pending_compaction_bytes_limit
+hard_pending_compaction_bytes_limit
+compaction_style
+compaction_pri
+compaction_options_universal
+compaction_options_fifo
+max_sequential_skip_in_iterations
+memtable_factory
+table_properties_collector_factories
+max_successive_merges
+optimize_filters_for_hits
+paranoid_file_checks
+force_consistency_checks
+report_bg_io_stats
+ttl
+periodic_compaction_seconds
+sample_for_compression
+default_write_temperature
+last_level_temperature
+default_temperature
+preclude_last_level_data_seconds
+preserve_internal_time_seconds
+enable_blob_files
+min_blob_size
+blob_file_size
+blob_compression_type
+enable_blob_garbage_collection
+blob_garbage_collection_age_cutoff
+blob_garbage_collection_force_threshold
+blob_compaction_readahead_size
+blob_file_starting_level
+blob_cache
+prepopulate_blob_cache
+memtable_protection_bytes_per_key
+persist_user_defined_timestamps
+block_protection_bytes_per_key
+bottommost_file_compaction_delay

missing_options_crash_test/aoptionsh_not.out

@@ -0,0 +1,10 @@
+p bloom_locality
+u compression_per_level
+u max_bytes_for_level_multiplier_additional
+u soft_pending_compaction_bytes_limit
+u hard_pending_compaction_bytes_limit
+optimize_filters_for_hits
+sample_for_compression
+u options
+u options_overrides
+report_bg_io_stats