default_write_temperature option (facebook#12388)

Summary:
Currently SST files that aren't applicable to last_level_temperature nor file_temperature_age_thresholds are written with temperature kUnknown, which is a little weird and doesn't support CF-based tiering. The default_temperature option only affects how kUnknown is interpreted for stats.

This change adds a new per-CF option default_write_temperature that determines the temperature of new SST files when those other options do not apply.

Also made a change to ignore last_level_temperature with FIFO compaction, because I found that could lead to an infinite loop in compaction.

Needed follow-up: Fix temperature handling with external file ingestion

Pull Request resolved: facebook#12388

Test Plan: unit tests extended appropriately. (Ignore whitespace changes when reviewing.)

Reviewed By: jowlyzhang

Differential Revision: D54266574

Pulled By: pdillinger

fbshipit-source-id: c9ec9a74dbf22be6e986f77f9689d05fea8ef0bb

db/column_family.cc

@@ -411,6 +411,13 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
           "periodic_compaction_seconds does not support FIFO compaction. You"
           "may want to set option TTL instead.");
     }
+    if (result.last_level_temperature != Temperature::kUnknown) {
+      ROCKS_LOG_WARN(
+          db_options.info_log.get(),
+          "last_level_temperature is ignored with FIFO compaction. Consider "
+          "CompactionOptionsFIFO::file_temperature_age_thresholds.");
+      result.last_level_temperature = Temperature::kUnknown;
+    }
   }
 
   // For universal compaction, `ttl` and `periodic_compaction_seconds` mean the

db/compaction/compaction_job.cc

@@ -1850,13 +1850,14 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
   // Pass temperature of the last level files to FileSystem.
   FileOptions fo_copy = file_options_;
   Temperature temperature = sub_compact->compaction->output_temperature();
-  // only set for the last level compaction and also it's not output to
-  // penultimate level (when preclude_last_level feature is enabled)
-  if (temperature == Temperature::kUnknown &&
+  Temperature last_level_temp =
+      sub_compact->compaction->mutable_cf_options()->last_level_temperature;
+  // Here last_level_temperature supersedes default_write_temperature, when
+  // enabled and applicable
+  if (last_level_temp != Temperature::kUnknown &&
       sub_compact->compaction->is_last_level() &&
       !sub_compact->IsCurrentPenultimateLevel()) {
-    temperature =
-        sub_compact->compaction->mutable_cf_options()->last_level_temperature;
+    temperature = last_level_temp;
   }
   fo_copy.temperature = temperature;
 

db/compaction/compaction_picker.cc

@@ -377,7 +377,8 @@ Compaction* CompactionPicker::CompactFiles(
       output_level, compact_options.output_file_size_limit,
       mutable_cf_options.max_compaction_bytes, output_path_id, compression_type,
       GetCompressionOptions(mutable_cf_options, vstorage, output_level),
-      Temperature::kUnknown, compact_options.max_subcompactions,
+      mutable_cf_options.default_write_temperature,
+      compact_options.max_subcompactions,
       /* grandparents */ {}, true);
   RegisterCompaction(c);
   return c;
@@ -664,7 +665,8 @@ Compaction* CompactionPicker::CompactRange(
         compact_range_options.target_path_id,
         GetCompressionType(vstorage, mutable_cf_options, output_level, 1),
         GetCompressionOptions(mutable_cf_options, vstorage, output_level),
-        Temperature::kUnknown, compact_range_options.max_subcompactions,
+        mutable_cf_options.default_write_temperature,
+        compact_range_options.max_subcompactions,
         /* grandparents */ {}, /* is manual */ true, trim_ts, /* score */ -1,
         /* deletion_compaction */ false, /* l0_files_might_overlap */ true,
         CompactionReason::kUnknown,
@@ -852,8 +854,9 @@ Compaction* CompactionPicker::CompactRange(
       GetCompressionType(vstorage, mutable_cf_options, output_level,
                          vstorage->base_level()),
       GetCompressionOptions(mutable_cf_options, vstorage, output_level),
-      Temperature::kUnknown, compact_range_options.max_subcompactions,
-      std::move(grandparents), /* is manual */ true, trim_ts, /* score */ -1,
+      mutable_cf_options.default_write_temperature,
+      compact_range_options.max_subcompactions, std::move(grandparents),
+      /* is manual */ true, trim_ts, /* score */ -1,
       /* deletion_compaction */ false, /* l0_files_might_overlap */ true,
       CompactionReason::kUnknown,
       compact_range_options.blob_garbage_collection_policy,

db/compaction/compaction_picker_fifo.cc

@@ -116,7 +116,8 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
   Compaction* c = new Compaction(
       vstorage, ioptions_, mutable_cf_options, mutable_db_options,
       std::move(inputs), 0, 0, 0, 0, kNoCompression,
-      mutable_cf_options.compression_opts, Temperature::kUnknown,
+      mutable_cf_options.compression_opts,
+      mutable_cf_options.default_write_temperature,
       /* max_subcompactions */ 0, {}, /* is manual */ false,
       /* trim_ts */ "", vstorage->CompactionScore(0),
       /* is deletion compaction */ true, /* l0_files_might_overlap */ true,
@@ -185,7 +186,8 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
             {comp_inputs}, 0, 16 * 1024 * 1024 /* output file size limit */,
             0 /* max compaction bytes, not applicable */,
             0 /* output path ID */, mutable_cf_options.compression,
-            mutable_cf_options.compression_opts, Temperature::kUnknown,
+            mutable_cf_options.compression_opts,
+            mutable_cf_options.default_write_temperature,
             0 /* max_subcompactions */, {}, /* is manual */ false,
             /* trim_ts */ "", vstorage->CompactionScore(0),
             /* is deletion compaction */ false,
@@ -280,7 +282,8 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
       /* target_file_size */ 0,
       /* max_compaction_bytes */ 0,
       /* output_path_id */ 0, kNoCompression,
-      mutable_cf_options.compression_opts, Temperature::kUnknown,
+      mutable_cf_options.compression_opts,
+      mutable_cf_options.default_write_temperature,
       /* max_subcompactions */ 0, {}, /* is manual */ false,
       /* trim_ts */ "", vstorage->CompactionScore(0),
       /* is deletion compaction */ true,
@@ -414,6 +417,7 @@ Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction(
   if (inputs[0].files.empty()) {
     return nullptr;
   }
+  assert(compaction_target_temp != Temperature::kLastTemperature);
 
   Compaction* c = new Compaction(
       vstorage, ioptions_, mutable_cf_options, mutable_db_options,

db/compaction/compaction_picker_level.cc

@@ -550,7 +550,7 @@ Compaction* LevelCompactionBuilder::GetCompaction() {
       GetCompressionType(vstorage_, mutable_cf_options_, output_level_,
                          vstorage_->base_level()),
       GetCompressionOptions(mutable_cf_options_, vstorage_, output_level_),
-      Temperature::kUnknown,
+      mutable_cf_options_.default_write_temperature,
       /* max_subcompactions */ 0, std::move(grandparents_), is_manual_,
       /* trim_ts */ "", start_level_score_, false /* deletion_compaction */,
       l0_files_might_overlap, compaction_reason_);

db/compaction/compaction_picker_universal.cc

@@ -870,7 +870,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns(
                                            output_level, 1, enable_compression),
                         GetCompressionOptions(mutable_cf_options_, vstorage_,
                                               output_level, enable_compression),
-                        Temperature::kUnknown,
+                        mutable_cf_options_.default_write_temperature,
                         /* max_subcompactions */ 0, grandparents,
                         /* is manual */ false, /* trim_ts */ "", score_,
                         false /* deletion_compaction */,
@@ -1204,7 +1204,7 @@ Compaction* UniversalCompactionBuilder::PickIncrementalForReduceSizeAmp(
                          true /* enable_compression */),
       GetCompressionOptions(mutable_cf_options_, vstorage_, output_level,
                             true /* enable_compression */),
-      Temperature::kUnknown,
+      mutable_cf_options_.default_write_temperature,
       /* max_subcompactions */ 0, /* grandparents */ {}, /* is manual */ false,
       /* trim_ts */ "", score_, false /* deletion_compaction */,
       /* l0_files_might_overlap */ true,
@@ -1347,7 +1347,7 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() {
       /* max_grandparent_overlap_bytes */ GetMaxOverlappingBytes(), path_id,
       GetCompressionType(vstorage_, mutable_cf_options_, output_level, 1),
       GetCompressionOptions(mutable_cf_options_, vstorage_, output_level),
-      Temperature::kUnknown,
+      mutable_cf_options_.default_write_temperature,
       /* max_subcompactions */ 0, grandparents, /* is manual */ false,
       /* trim_ts */ "", score_, false /* deletion_compaction */,
       /* l0_files_might_overlap */ true,
@@ -1440,7 +1440,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionWithSortedRunRange(
                          true /* enable_compression */),
       GetCompressionOptions(mutable_cf_options_, vstorage_, output_level,
                             true /* enable_compression */),
-      Temperature::kUnknown,
+      mutable_cf_options_.default_write_temperature,
       /* max_subcompactions */ 0, /* grandparents */ {}, /* is manual */ false,
       /* trim_ts */ "", score_, false /* deletion_compaction */,
       /* l0_files_might_overlap */ true, compaction_reason);

db/db_compaction_test.cc

@@ -9164,66 +9164,104 @@ TEST_F(DBCompactionTest, CompactionWithChecksumHandoffManifest2) {
 }
 
 TEST_F(DBCompactionTest, FIFOChangeTemperature) {
-  Options options = CurrentOptions();
-  options.compaction_style = kCompactionStyleFIFO;
-  options.num_levels = 1;
-  options.max_open_files = -1;
-  options.level0_file_num_compaction_trigger = 2;
-  options.create_if_missing = true;
-  CompactionOptionsFIFO fifo_options;
-  fifo_options.file_temperature_age_thresholds = {{Temperature::kCold, 1000}};
-  fifo_options.max_table_files_size = 100000000;
-  options.compaction_options_fifo = fifo_options;
-  env_->SetMockSleep();
-  Reopen(options);
+  for (bool write_time_default : {false, true}) {
+    SCOPED_TRACE("write time default? " + std::to_string(write_time_default));
 
-  int total_cold = 0;
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "NewWritableFile::FileOptions.temperature", [&](void* arg) {
-        Temperature temperature = *(static_cast<Temperature*>(arg));
-        if (temperature == Temperature::kCold) {
-          total_cold++;
-        }
-      });
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+    Options options = CurrentOptions();
+    options.compaction_style = kCompactionStyleFIFO;
+    options.num_levels = 1;
+    options.max_open_files = -1;
+    options.level0_file_num_compaction_trigger = 2;
+    options.create_if_missing = true;
+    CompactionOptionsFIFO fifo_options;
+    fifo_options.file_temperature_age_thresholds = {{Temperature::kCold, 1000}};
+    fifo_options.max_table_files_size = 100000000;
+    options.compaction_options_fifo = fifo_options;
+    env_->SetMockSleep();
+    if (write_time_default) {
+      options.default_write_temperature = Temperature::kWarm;
+    }
+    // Should be ignored (TODO: fail?)
+    options.last_level_temperature = Temperature::kHot;
+    Reopen(options);
 
-  // The file system does not support checksum handoff. The check
-  // will be ignored.
-  ASSERT_OK(Put(Key(0), "value1"));
-  env_->MockSleepForSeconds(800);
-  ASSERT_OK(Put(Key(2), "value2"));
-  ASSERT_OK(Flush());
+    int total_cold = 0;
+    int total_warm = 0;
+    int total_hot = 0;
+    int total_unknown = 0;
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+        "NewWritableFile::FileOptions.temperature", [&](void* arg) {
+          Temperature temperature = *(static_cast<Temperature*>(arg));
+          if (temperature == Temperature::kCold) {
+            total_cold++;
+          } else if (temperature == Temperature::kWarm) {
+            total_warm++;
+          } else if (temperature == Temperature::kHot) {
+            total_hot++;
+          } else {
+            assert(temperature == Temperature::kUnknown);
+            total_unknown++;
+          }
+        });
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
 
-  ASSERT_OK(Put(Key(0), "value1"));
-  env_->MockSleepForSeconds(800);
-  ASSERT_OK(Put(Key(2), "value2"));
-  ASSERT_OK(Flush());
+    // The file system does not support checksum handoff. The check
+    // will be ignored.
+    ASSERT_OK(Put(Key(0), "value1"));
+    env_->MockSleepForSeconds(800);
+    ASSERT_OK(Put(Key(2), "value2"));
+    ASSERT_OK(Flush());
 
-  ASSERT_OK(Put(Key(0), "value1"));
-  env_->MockSleepForSeconds(800);
-  ASSERT_OK(Put(Key(2), "value2"));
-  ASSERT_OK(Flush());
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    ASSERT_OK(Put(Key(0), "value1"));
+    env_->MockSleepForSeconds(800);
+    ASSERT_OK(Put(Key(2), "value2"));
+    ASSERT_OK(Flush());
 
-  ASSERT_OK(Put(Key(0), "value1"));
-  env_->MockSleepForSeconds(800);
-  ASSERT_OK(Put(Key(2), "value2"));
-  ASSERT_OK(Flush());
+    ASSERT_OK(Put(Key(0), "value1"));
+    env_->MockSleepForSeconds(800);
+    ASSERT_OK(Put(Key(2), "value2"));
+    ASSERT_OK(Flush());
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
 
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    if (write_time_default) {
+      // Also test dynamic option change
+      ASSERT_OK(db_->SetOptions({{"default_write_temperature", "kHot"}}));
+    }
 
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+    ASSERT_OK(Put(Key(0), "value1"));
+    env_->MockSleepForSeconds(800);
+    ASSERT_OK(Put(Key(2), "value2"));
+    ASSERT_OK(Flush());
 
-  ColumnFamilyMetaData metadata;
-  db_->GetColumnFamilyMetaData(&metadata);
-  ASSERT_EQ(4, metadata.file_count);
-  ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature);
-  ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[1].temperature);
-  ASSERT_EQ(Temperature::kCold, metadata.levels[0].files[2].temperature);
-  ASSERT_EQ(Temperature::kCold, metadata.levels[0].files[3].temperature);
-  ASSERT_EQ(2, total_cold);
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
 
-  Destroy(options);
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+
+    ColumnFamilyMetaData metadata;
+    db_->GetColumnFamilyMetaData(&metadata);
+    ASSERT_EQ(4, metadata.file_count);
+    if (write_time_default) {
+      ASSERT_EQ(Temperature::kHot, metadata.levels[0].files[0].temperature);
+      ASSERT_EQ(Temperature::kWarm, metadata.levels[0].files[1].temperature);
+      // Includes obsolete/deleted files moved to cold
+      ASSERT_EQ(total_warm, 3);
+      ASSERT_EQ(total_hot, 1);
+      // Includes non-SST DB files
+      ASSERT_GT(total_unknown, 0);
+    } else {
+      ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature);
+      ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[1].temperature);
+      ASSERT_EQ(total_warm, 0);
+      ASSERT_EQ(total_hot, 0);
+      // Includes non-SST DB files
+      ASSERT_GT(total_unknown, 4);
+    }
+    ASSERT_EQ(Temperature::kCold, metadata.levels[0].files[2].temperature);
+    ASSERT_EQ(Temperature::kCold, metadata.levels[0].files[3].temperature);
+    ASSERT_EQ(2, total_cold);
+
+    Destroy(options);
+  }
 }
 
 TEST_F(DBCompactionTest, DisableMultiManualCompaction) {

db/db_impl/db_impl_compaction_flush.cc

@@ -1868,7 +1868,8 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
         ,
         LLONG_MAX /* max compaction bytes, not applicable */,
         0 /* output path ID, not applicable */, mutable_cf_options.compression,
-        mutable_cf_options.compression_opts, Temperature::kUnknown,
+        mutable_cf_options.compression_opts,
+        mutable_cf_options.default_write_temperature,
         0 /* max_subcompactions, not applicable */,
         {} /* grandparents, not applicable */, false /* is manual */,
         "" /* trim_ts */, -1 /* score, not applicable */,