Send datasize stats by channel along with total (ydb-platform#11675)

zverevgeny · Jan 5, 2025 · de56f6e · de56f6e
1 parent 290c498
commit de56f6e
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 0 deletions.
diff --git a/ydb/core/tx/columnshard/common/blob.h b/ydb/core/tx/columnshard/common/blob.h
@@ -105,6 +105,10 @@ class TUnifiedBlobId {
         return Id.BlobId.BlobSize();
     }
 
+    ui32 Channel() const {
+        return Id.BlobId.Channel();
+    }
+
     TLogoBlobID GetLogoBlobId() const {
         return Id.BlobId;
     }

diff --git a/ydb/core/tx/columnshard/counters/aggregation/table_stats.h b/ydb/core/tx/columnshard/counters/aggregation/table_stats.h
@@ -41,6 +41,14 @@ class TTableStatsBuilder {
         auto activeStats = ColumnEngine.GetTotalStats().Active();
         tableStats.SetRowCount(activeStats.Rows);
         tableStats.SetDataSize(activeStats.Bytes);
+        for (ui32 ch = 0; ch < activeStats.ByChannel.size(); ch++) {
+            ui64 dataSize = activeStats.ByChannel[ch];
+            if (dataSize > 0) {
+                auto item = tableStats.AddChannels();
+                item->SetChannel(ch);
+                item->SetDataSize(dataSize);
+            }
+        }
     }
 };
 

diff --git a/ydb/core/tx/columnshard/engines/column_engine.h b/ydb/core/tx/columnshard/engines/column_engine.h
@@ -74,6 +74,7 @@ class TColumnEngineStats {
         i64 Rows = 0;
         i64 Bytes = 0;
         i64 RawBytes = 0;
+        std::vector<i64> ByChannel;
 
         TString DebugString() const {
             return TStringBuilder() << "portions=" << Portions << ";blobs=" << Blobs << ";rows=" << Rows << ";bytes=" << Bytes
@@ -92,6 +93,10 @@ class TColumnEngineStats {
             result.Rows = kff * Rows;
             result.Bytes = kff * Bytes;
             result.RawBytes = kff * RawBytes;
+            result.ByChannel.reserve(ByChannel.size());
+            for (ui64 channelBytes: ByChannel) {
+                result.ByChannel.push_back(channelBytes * kff);
+            }
             return result;
         }
 
@@ -105,6 +110,12 @@ class TColumnEngineStats {
             Rows = SumVerifiedPositive(Rows, item.Rows);
             Bytes = SumVerifiedPositive(Bytes, item.Bytes);
             RawBytes = SumVerifiedPositive(RawBytes, item.RawBytes);
+            if (ByChannel.size() < item.ByChannel.size()) {
+                ByChannel.resize(item.ByChannel.size());
+            }
+            for (ui32 ch = 0; ch < item.ByChannel.size(); ch++) {
+                ByChannel[ch] = SumVerifiedPositive(ByChannel[ch], item.ByChannel[ch]);
+            }
             return *this;
         }
     };

diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp
@@ -98,6 +98,19 @@ void TColumnEngineForLogs::UpdatePortionStats(
     TColumnEngineStats& engineStats, const TPortionInfo& portionInfo, EStatsUpdateType updateType, const TPortionInfo* exPortionInfo) const {
     TColumnEngineStats::TPortionsStats deltaStats = DeltaStats(portionInfo);
 
+    ui64 totalBlobsSize = 0;
+    ui32 blobCount = portionInfo.GetBlobIdsCount();
+    for (ui32 i = 0; i < blobCount; i++) {
+        const auto& blob = portionInfo.GetBlobId(i);
+        ui32 channel = blob.Channel();
+        if (deltaStats.ByChannel.size() <= channel) {
+            deltaStats.ByChannel.resize(channel + 1);
+        }
+        deltaStats.ByChannel[channel] += blob.BlobSize();
+        totalBlobsSize += blob.BlobSize();
+    }
+    AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "update_portion")("blobs_size", totalBlobsSize)("portion_bytes", deltaStats.Bytes)("portion_raw_bytes", deltaStats.RawBytes);
+
     Y_ABORT_UNLESS(!exPortionInfo || exPortionInfo->GetMeta().Produced != TPortionMeta::EProduced::UNSPECIFIED);
     Y_ABORT_UNLESS(portionInfo.GetMeta().Produced != TPortionMeta::EProduced::UNSPECIFIED);