Fix background_pool_size not take effect and BackgroundProcessingPool::getThreadIds may misses some thread_ids.

dbms/CMakeLists.txt

@@ -206,6 +206,7 @@ target_link_libraries (dbms
     ${RE2_ST_LIBRARY}
     ${OPENSSL_CRYPTO_LIBRARY}
     ${BTRIE_LIBRARIES}
+    absl::synchronization
 )
 
 if (NOT USE_INTERNAL_RE2_LIBRARY)

dbms/src/Encryption/RateLimiter.cpp

@@ -523,6 +523,7 @@ void IORateLimiter::setBackgroundThreadIds(std::vector<pid_t> thread_ids)
 {
     std::lock_guard lock(bg_thread_ids_mtx);
     bg_thread_ids.swap(thread_ids);
+    LOG_FMT_INFO(log, "bg_thread_ids {} => {}", bg_thread_ids.size(), bg_thread_ids);
 }
 
 std::pair<Int64, Int64> IORateLimiter::getReadWriteBytes(const std::string & fname [[maybe_unused]])

dbms/src/Interpreters/Context.cpp

@@ -1548,11 +1548,11 @@ FileProviderPtr Context::getFileProvider() const
     return shared->file_provider;
 }
 
-void Context::initializeRateLimiter(Poco::Util::AbstractConfiguration & config)
+void Context::initializeRateLimiter(Poco::Util::AbstractConfiguration & config, BackgroundProcessingPool & bg_pool, BackgroundProcessingPool & blockable_bg_pool) const
 {
     getIORateLimiter().init(config);
-    auto tids = getBackgroundPool().getThreadIds();
-    auto blockable_tids = getBlockableBackgroundPool().getThreadIds();
+    auto tids = bg_pool.getThreadIds();
+    auto blockable_tids = blockable_bg_pool.getThreadIds();
     tids.insert(tids.end(), blockable_tids.begin(), blockable_tids.end());
     getIORateLimiter().setBackgroundThreadIds(tids);
 }

dbms/src/Interpreters/Context.h

@@ -404,7 +404,7 @@ class Context
     void initializeFileProvider(KeyManagerPtr key_manager, bool enable_encryption);
     FileProviderPtr getFileProvider() const;
 
-    void initializeRateLimiter(Poco::Util::AbstractConfiguration & config);
+    void initializeRateLimiter(Poco::Util::AbstractConfiguration & config, BackgroundProcessingPool & bg_pool, BackgroundProcessingPool & blockable_bg_pool) const;
     WriteLimiterPtr getWriteLimiter() const;
     ReadLimiterPtr getReadLimiter() const;
     IORateLimiter & getIORateLimiter() const;

dbms/src/Server/Server.cpp

@@ -1215,8 +1215,22 @@ int Server::main(const std::vector<std::string> & /*args*/)
     /// Init TiFlash metrics.
     global_context->initializeTiFlashMetrics();
 
-    /// Init Rate Limiter
-    global_context->initializeRateLimiter(config());
+    /// Initialize users config reloader.
+    auto users_config_reloader = UserConfig::parseSettings(config(), config_path, global_context, log);
+
+    /// Load global settings from default_profile and system_profile.
+    /// It internally depends on UserConfig::parseSettings.
+    global_context->setDefaultProfiles(config());
+    Settings & settings = global_context->getSettingsRef();
+
+    /// Initialize the background thread pool.
+    /// It internally depends on settings.background_pool_size,
+    /// so must be called after settings has been load.
+    auto & bg_pool = global_context->getBackgroundPool();
+    auto & blockable_bg_pool = global_context->getBlockableBackgroundPool();
+
+    /// Initialize RateLimiter.
+    global_context->initializeRateLimiter(config(), bg_pool, blockable_bg_pool);
 
     /// Initialize main config reloader.
     auto main_config_reloader = std::make_unique<ConfigReloader>(
@@ -1230,9 +1244,6 @@ int Server::main(const std::vector<std::string> & /*args*/)
         },
         /* already_loaded = */ true);
 
-    /// Initialize users config reloader.
-    auto users_config_reloader = UserConfig::parseSettings(config(), config_path, global_context, log);
-
     /// Reload config in SYSTEM RELOAD CONFIG query.
     global_context->setConfigReloadCallback([&]() {
         main_config_reloader->reload();
@@ -1254,10 +1265,6 @@ int Server::main(const std::vector<std::string> & /*args*/)
     bool use_l0_opt = config().getBool("l0_optimize", false);
     global_context->setUseL0Opt(use_l0_opt);
 
-    /// Load global settings from default_profile and system_profile.
-    global_context->setDefaultProfiles(config());
-    Settings & settings = global_context->getSettingsRef();
-
     /// Size of cache for marks (index of MergeTree family of tables). It is necessary.
     size_t mark_cache_size = config().getUInt64("mark_cache_size", DEFAULT_MARK_CACHE_SIZE);
     if (mark_cache_size)

dbms/src/Storages/BackgroundProcessingPool.cpp

@@ -29,6 +29,15 @@
 #ifdef __linux__
 #include <sys/syscall.h>
 #include <unistd.h>
+inline static pid_t gettid()
+{
+    return syscall(SYS_gettid);
+}
+#elif
+inline static pid_t gettid()
+{
+    return -1;
+}
 #endif
 
 namespace CurrentMetrics
@@ -76,6 +85,7 @@ void BackgroundProcessingPool::TaskInfo::wake()
 
 BackgroundProcessingPool::BackgroundProcessingPool(int size_)
     : size(size_)
+    , thread_ids_counter(size_)
 {
     LOG_FMT_INFO(&Poco::Logger::get("BackgroundProcessingPool"), "Create BackgroundProcessingPool with {} threads", size);
 
@@ -140,9 +150,7 @@ void BackgroundProcessingPool::threadFunction()
         const auto name = "BkgPool" + std::to_string(tid++);
         setThreadName(name.data());
         is_background_thread = true;
-#ifdef __linux__
-        addThreadId(syscall(SYS_gettid));
-#endif
+        addThreadId(gettid());
     }
 
     MemoryTracker memory_tracker;
@@ -272,14 +280,23 @@ void BackgroundProcessingPool::threadFunction()
 
 std::vector<pid_t> BackgroundProcessingPool::getThreadIds()
 {
+    thread_ids_counter.Wait();
     std::lock_guard lock(thread_ids_mtx);
+    if (thread_ids.size() != size)
+    {
+        LOG_FMT_ERROR(&Poco::Logger::get("BackgroundProcessingPool"), "thread_ids.size is {}, but {} is required", thread_ids.size(), size);
+        throw Exception("Background threads' number not match");
+    }
     return thread_ids;
 }
 
 void BackgroundProcessingPool::addThreadId(pid_t tid)
 {
-    std::lock_guard lock(thread_ids_mtx);
-    thread_ids.push_back(tid);
+    {
+        std::lock_guard lock(thread_ids_mtx);
+        thread_ids.push_back(tid);
+    }
+    thread_ids_counter.DecrementCount();
 }
 
 } // namespace DB

dbms/src/Storages/BackgroundProcessingPool.h

@@ -17,6 +17,7 @@
 #include <Core/Types.h>
 #include <Poco/Event.h>
 #include <Poco/Timestamp.h>
+#include <absl/synchronization/blocking_counter.h>
 
 #include <atomic>
 #include <condition_variable>
@@ -117,6 +118,7 @@ class BackgroundProcessingPool
     Threads threads;
     std::vector<pid_t> thread_ids; // Linux Thread ID
     std::mutex thread_ids_mtx;
+    absl::BlockingCounter thread_ids_counter;
 
     std::atomic<bool> shutdown{false};
     std::condition_variable wake_event;

libs/libcommon/include/common/logger_useful.h

@@ -18,6 +18,7 @@
 
 #include <Poco/Logger.h>
 #include <fmt/format.h>
+#include <fmt/ranges.h>
 
 #ifndef QUERY_PREVIEW_LENGTH
 #define QUERY_PREVIEW_LENGTH 160