chunks count for limit in compaction (ydb-platform#10812)

zverevgeny · Jan 8, 2025 · 3750139 · 3750139
1 parent 9aa2369
commit 3750139
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 1 deletion.
diff --git a/ydb/core/tx/columnshard/counters/portions.cpp b/ydb/core/tx/columnshard/counters/portions.cpp
@@ -30,6 +30,7 @@ void TSimplePortionsGroupInfo::AddPortion(const TPortionInfo& p) {
     RawBytes += p.GetTotalRawBytes();
     Count += 1;
     RecordsCount += p.NumRows();
+    ChunksCount += p.GetChunksCount();
 }
 
 void TSimplePortionsGroupInfo::RemovePortion(const std::shared_ptr<TPortionInfo>& p) {
@@ -41,10 +42,12 @@ void TSimplePortionsGroupInfo::RemovePortion(const TPortionInfo& p) {
     RawBytes -= p.GetTotalRawBytes();
     Count -= 1;
     RecordsCount -= p.NumRows();
+    ChunksCount -= p.GetChunksCount();
     AFL_VERIFY(RawBytes >= 0);
     AFL_VERIFY(BlobBytes >= 0);
     AFL_VERIFY(Count >= 0);
     AFL_VERIFY(RecordsCount >= 0);
+    AFL_VERIFY(ChunksCount >= 0);
 }
 
 }   // namespace NKikimr::NOlap
diff --git a/ydb/core/tx/columnshard/counters/portions.h b/ydb/core/tx/columnshard/counters/portions.h
@@ -15,6 +15,7 @@ class TSimplePortionsGroupInfo {
     YDB_READONLY(i64, RawBytes, 0);
     YDB_READONLY(i64, Count, 0);
     YDB_READONLY(i64, RecordsCount, 0);
+    YDB_READONLY(i64, ChunksCount, 0);
 
 public:
     NJson::TJsonValue SerializeToJson() const {
@@ -23,6 +24,7 @@ class TSimplePortionsGroupInfo {
         result.InsertValue("raw_bytes", RawBytes);
         result.InsertValue("count", Count);
         result.InsertValue("records_count", RecordsCount);
+        result.InsertValue("chunks_count", ChunksCount);
         return result;
     }
 
@@ -45,6 +47,7 @@ class TSimplePortionsGroupInfo {
         result.RawBytes = RawBytes + item.RawBytes;
         result.Count = Count + item.Count;
         result.RecordsCount = RecordsCount + item.RecordsCount;
+        result.ChunksCount = ChunksCount + item.ChunksCount;
         return result;
     }
 

diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.h b/ydb/core/tx/columnshard/engines/portions/portion_info.h
@@ -141,6 +141,10 @@ class TPortionInfo {
 
     bool NeedShardingFilter(const TGranuleShardingInfo& shardingInfo) const;
 
+    ui64 GetChunksCount() const {
+        return Records.size() + Indexes.size();
+    }
+
     NSplitter::TEntityGroups GetEntityGroupsByStorageId(
         const TString& specialTier, const IStoragesManager& storages, const TIndexInfo& indexInfo) const;
 

diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/abstract.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/abstract.h
@@ -251,7 +251,11 @@ class TCompactionTaskData {
     }
 
     bool CanTakeMore() const {
-        return MemoryUsage < (((ui64)512) << 20) && Portions.size() < 10000;
+        if (Portions.size() <= 1) {
+            return true;
+        }
+        return MemoryUsage < (((ui64)512) << 20) && CurrentLevelPortionsInfo.GetChunksCount() + TargetLevelPortionsInfo.GetChunksCount() < 100000
+            && Portions.size() < 10000;
     }
 
     TCompactionTaskData(const ui64 targetCompactionLevel)