Add shred stub into BSC (#13310)

ydb-platform · Jan 13, 2025 · e9ac0af · e9ac0af
1 parent fc1779f
commit e9ac0af
Show file tree

Hide file tree

Showing 14 changed files with 169 additions and 3 deletions.
diff --git a/ydb/core/base/blobstorage.h b/ydb/core/base/blobstorage.h
@@ -849,6 +849,8 @@ struct TEvBlobStorage {
         EvControllerValidateConfigResponse          = 0x10031626,
         EvControllerReplaceConfigRequest            = 0x10031627,
         EvControllerReplaceConfigResponse           = 0x10031628,
+        EvControllerShredRequest                    = 0x10031629,
+        EvControllerShredResponse                   = 0x1003162a,
 
         // BSC interface result section
         EvControllerNodeServiceSetUpdate            = 0x10031802,
@@ -2494,6 +2496,8 @@ struct TEvBlobStorage {
     struct TEvControllerValidateConfigResponse;
     struct TEvControllerReplaceConfigRequest;
     struct TEvControllerReplaceConfigResponse;
+    struct TEvControllerShredRequest;
+    struct TEvControllerShredResponse;
 
     struct TEvMonStreamQuery;
     struct TEvMonStreamActorDeathNote;

diff --git a/ydb/core/blobstorage/base/blobstorage_shred_events.h b/ydb/core/blobstorage/base/blobstorage_shred_events.h
@@ -0,0 +1,24 @@
+#pragma once
+
+#include "defs.h"
+
+#include <ydb/core/base/blobstorage.h>
+#include <ydb/core/protos/blobstorage.pb.h>
+
+namespace NKikimr {
+
+    struct TEvBlobStorage::TEvControllerShredRequest : TEventPB<TEvControllerShredRequest,
+            NKikimrBlobStorage::TEvControllerShredRequest, TEvBlobStorage::EvControllerShredRequest> {
+        TEvControllerShredRequest() = default;
+
+        TEvControllerShredRequest(ui64 generation) {
+            Record.SetGeneration(generation);
+        }
+    };
+
+    struct TEvBlobStorage::TEvControllerShredResponse : TEventPB<TEvControllerShredResponse,
+            NKikimrBlobStorage::TEvControllerShredResponse, TEvBlobStorage::EvControllerShredResponse> {
+        TEvControllerShredResponse() = default;
+    };
+
+}
diff --git a/ydb/core/blobstorage/ut_blobstorage/shred.cpp b/ydb/core/blobstorage/ut_blobstorage/shred.cpp
@@ -0,0 +1,40 @@
+#include <ydb/core/blobstorage/ut_blobstorage/lib/env.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+Y_UNIT_TEST_SUITE(Shred) {
+
+    Y_UNIT_TEST(Basic) {
+        TEnvironmentSetup env{{}};
+        env.CreateBoxAndPool(1, 1);
+        env.Sim(TDuration::Seconds(5));
+
+        {
+            const TActorId self = env.Runtime->AllocateEdgeActor(env.Settings.ControllerNodeId, __FILE__, __LINE__);
+            auto ev = std::make_unique<TEvBlobStorage::TEvControllerShredRequest>();
+            env.Runtime->SendToPipe(env.TabletId, self, ev.release(), 0, TTestActorSystem::GetPipeConfigWithRetries());
+            auto r = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvControllerShredResponse>(self);
+            UNIT_ASSERT(!r->Get()->Record.HasCurrentGeneration());
+        }
+
+        {
+            const TActorId self = env.Runtime->AllocateEdgeActor(env.Settings.ControllerNodeId, __FILE__, __LINE__);
+            auto ev = std::make_unique<TEvBlobStorage::TEvControllerShredRequest>(1);
+            env.Runtime->SendToPipe(env.TabletId, self, ev.release(), 0, TTestActorSystem::GetPipeConfigWithRetries());
+            auto r = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvControllerShredResponse>(self);
+            UNIT_ASSERT_VALUES_EQUAL(r->Get()->Record.GetCurrentGeneration(), 1);
+            UNIT_ASSERT_VALUES_EQUAL(r->Get()->Record.GetCompleted(), false);
+            UNIT_ASSERT_VALUES_EQUAL(r->Get()->Record.GetProgress10k(), 0);
+        }
+
+        {
+            const TActorId self = env.Runtime->AllocateEdgeActor(env.Settings.ControllerNodeId, __FILE__, __LINE__);
+            auto ev = std::make_unique<TEvBlobStorage::TEvControllerShredRequest>(0);
+            env.Runtime->SendToPipe(env.TabletId, self, ev.release(), 0, TTestActorSystem::GetPipeConfigWithRetries());
+            auto r = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvControllerShredResponse>(self);
+            UNIT_ASSERT_VALUES_EQUAL(r->Get()->Record.GetCurrentGeneration(), 1);
+            UNIT_ASSERT_VALUES_EQUAL(r->Get()->Record.GetCompleted(), false);
+            UNIT_ASSERT_VALUES_EQUAL(r->Get()->Record.GetProgress10k(), 0);
+        }
+    }
+
+}
diff --git a/ydb/core/blobstorage/ut_blobstorage/ya.make b/ydb/core/blobstorage/ut_blobstorage/ya.make
@@ -41,6 +41,7 @@ SRCS(
     recovery.cpp
     sanitize_groups.cpp
     scrub_fast.cpp
+    shred.cpp
     snapshots.cpp
     space_check.cpp
     sync.cpp

diff --git a/ydb/core/mind/bscontroller/bsc.cpp b/ydb/core/mind/bscontroller/bsc.cpp
@@ -431,6 +431,7 @@ STFUNC(TBlobStorageController::StateWork) {
         hFunc(TEvTabletPipe::TEvClientConnected, ConsoleInteraction->Handle);
         hFunc(TEvTabletPipe::TEvClientDestroyed, ConsoleInteraction->Handle);
         hFunc(TEvBlobStorage::TEvGetBlockResult, ConsoleInteraction->Handle);
+        fFunc(TEvBlobStorage::EvControllerShredRequest, EnqueueIncomingEvent);
         default:
             if (!HandleDefaultEvents(ev, SelfId())) {
                 STLOG(PRI_ERROR, BS_CONTROLLER, BSC06, "StateWork unexpected event", (Type, type),
@@ -508,6 +509,7 @@ ui32 TBlobStorageController::GetEventPriority(IEventHandle *ev) {
         case TEvBlobStorage::EvControllerScrubQuantumFinished:         return 2;
         case TEvBlobStorage::EvControllerScrubReportQuantumInProgress: return 2;
         case TEvBlobStorage::EvControllerUpdateNodeDrives:             return 2;
+        case TEvBlobStorage::EvControllerShredRequest:                 return 2;
 
         // hive-related commands
         case TEvBlobStorage::EvControllerSelectGroups:                 return 4;

diff --git a/ydb/core/mind/bscontroller/defs.h b/ydb/core/mind/bscontroller/defs.h
@@ -13,6 +13,7 @@
 #include <ydb/core/base/tablet_pipe.h>
 #include <ydb/core/blobstorage/base/blobstorage_events.h>
 #include <ydb/core/blobstorage/base/blobstorage_console_events.h>
+#include <ydb/core/blobstorage/base/blobstorage_shred_events.h>
 #include <ydb/core/blobstorage/base/blobstorage_vdiskid.h>
 #include <ydb/core/blobstorage/base/utility.h>
 #include <ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h>

diff --git a/ydb/core/mind/bscontroller/impl.h b/ydb/core/mind/bscontroller/impl.h
@@ -71,6 +71,7 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
     class TTxUpdateLastSeenReady;
     class TTxUpdateNodeDrives;
     class TTxUpdateNodeDisconnectTimestamp;
+    class TTxUpdateShred;
 
     class TVSlotInfo;
     class TPDiskInfo;
@@ -1532,6 +1533,7 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
     TString YamlConfig;
     ui32 ConfigVersion = 0;
     TBackoffTimer GetBlockBackoff{1, 1000};
+    NKikimrBlobStorage::TShredState ShredState;
 
     THashMap<TPDiskId, std::reference_wrapper<const NKikimrBlobStorage::TNodeWardenServiceSet::TPDisk>> StaticPDiskMap;
     THashMap<TPDiskId, ui32> StaticPDiskSlotUsage;
@@ -1836,6 +1838,7 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
     void ForwardToSystemViewsCollector(STATEFN_SIG);
     void Handle(TEvPrivate::TEvUpdateSystemViews::TPtr &ev);
     void Handle(TEvBlobStorage::TEvGetBlockResult::TPtr &ev);
+    void Handle(TEvBlobStorage::TEvControllerShredRequest::TPtr ev);
 
     ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
     // Scrub handling
@@ -2075,6 +2078,7 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
             hFunc(TEvBlobStorage::TEvControllerGroupDecommittedNotify, Handle);
             cFunc(TEvPrivate::EvScrub, ScrubState.HandleTimer);
             cFunc(TEvPrivate::EvVSlotReadyUpdate, VSlotReadyUpdate);
+            hFunc(TEvBlobStorage::TEvControllerShredRequest, Handle);
         }
 
         if (const TDuration time = TDuration::Seconds(timer.Passed()); time >= TDuration::MilliSeconds(100)) {

diff --git a/ydb/core/mind/bscontroller/load_everything.cpp b/ydb/core/mind/bscontroller/load_everything.cpp
@@ -93,6 +93,11 @@ class TBlobStorageController::TTxLoadEverything : public TTransactionBase<TBlobS
                 Self->TryToRelocateBrokenDisksLocallyFirst = state.GetValue<T::TryToRelocateBrokenDisksLocallyFirst>();
                 Self->YamlConfig = state.GetValue<T::YamlConfig>();
                 Self->ConfigVersion = state.GetValue<T::ConfigVersion>();
+                if (state.HaveValue<T::ShredState>()) {
+                    TString buffer = state.GetValue<T::ShredState>();
+                    const bool success = Self->ShredState.ParseFromString(buffer);
+                    Y_ABORT_UNLESS(success);
+                }
             }
         }
 

diff --git a/ydb/core/mind/bscontroller/scheme.h b/ydb/core/mind/bscontroller/scheme.h
@@ -109,13 +109,14 @@ struct Schema : NIceDb::Schema {
         struct TryToRelocateBrokenDisksLocallyFirst : Column<23, NScheme::NTypeIds::Bool> { static constexpr Type Default = false; };
         struct YamlConfig : Column<24, NScheme::NTypeIds::String> {};
         struct ConfigVersion : Column<25, NScheme::NTypeIds::Uint32> { static constexpr Type Default = 0; };
+        struct ShredState : Column<26, NScheme::NTypeIds::String> {};
 
         using TKey = TableKey<FixedKey>;
         using TColumns = TableColumns<FixedKey, NextGroupID, SchemaVersion, NextOperationLogIndex, DefaultMaxSlots,
               InstanceId, SelfHealEnable, DonorModeEnable, ScrubPeriodicity, SerialManagementStage, NextStoragePoolId,
               PDiskSpaceMarginPromille, GroupReserveMin, GroupReservePart, MaxScrubbedDisksAtOnce, PDiskSpaceColorBorder,
               GroupLayoutSanitizer, NextVirtualGroupId, AllowMultipleRealmsOccupation, CompatibilityInfo,
-              UseSelfHealLocalPolicy, TryToRelocateBrokenDisksLocallyFirst, YamlConfig, ConfigVersion>;
+              UseSelfHealLocalPolicy, TryToRelocateBrokenDisksLocallyFirst, YamlConfig, ConfigVersion, ShredState>;
     };
 
     struct VSlot : Table<5> {

diff --git a/ydb/core/mind/bscontroller/shred.cpp b/ydb/core/mind/bscontroller/shred.cpp
@@ -0,0 +1,63 @@
+#include "impl.h"
+
+namespace NKikimr::NBsController {
+
+    class TBlobStorageController::TTxUpdateShred : public TTransactionBase<TBlobStorageController> {
+        const NKikimrBlobStorage::TEvControllerShredRequest Request;
+        const TActorId Sender;
+        const ui64 Cookie;
+        const TActorId InterconnectSession;
+
+    public:
+        TTxUpdateShred(TBlobStorageController *controller, TEvBlobStorage::TEvControllerShredRequest::TPtr ev)
+            : TTransactionBase(controller)
+            , Request(std::move(ev->Get()->Record))
+            , Sender(ev->Sender)
+            , Cookie(ev->Cookie)
+            , InterconnectSession(ev->InterconnectSession)
+        {}
+
+        TTxType GetTxType() const override { return NBlobStorageController::TXTYPE_UPDATE_SHRED; }
+
+        bool Execute(TTransactionContext& txc, const TActorContext&) override {
+            auto& current = Self->ShredState;
+            if (Request.HasGeneration() && (!current.HasGeneration() || current.GetGeneration() < Request.GetGeneration())) {
+                // reset shred state to initial one with newly provided generation
+                current.SetGeneration(Request.GetGeneration());
+                current.SetCompleted(false);
+                current.SetProgress10k(0);
+
+                // serialize it to string and update database
+                TString buffer;
+                const bool success = current.SerializeToString(&buffer);
+                Y_ABORT_UNLESS(success);
+                NIceDb::TNiceDb db(txc.DB);
+                db.Table<Schema::State>().Key(true).Update<Schema::State::ShredState>(buffer);
+            }
+
+            return true;
+        }
+
+        void Complete(const TActorContext&) override {
+            auto ev = std::make_unique<TEvBlobStorage::TEvControllerShredResponse>();
+            auto& r = ev->Record;
+            const auto& current = Self->ShredState;
+            if (current.HasGeneration()) {
+                r.SetCurrentGeneration(current.GetGeneration());
+                r.SetCompleted(current.GetCompleted());
+                r.SetProgress10k(current.GetProgress10k());
+            }
+
+            auto h = std::make_unique<IEventHandle>(Sender, Self->SelfId(), ev.release(), 0, Cookie);
+            if (InterconnectSession) {
+                h->Rewrite(TEvInterconnect::EvForward, InterconnectSession);
+            }
+            TActivationContext::Send(h.release());
+        }
+    };
+
+    void TBlobStorageController::Handle(TEvBlobStorage::TEvControllerShredRequest::TPtr ev) {
+        Execute(new TTxUpdateShred(this, ev));
+    }
+
+} // NKikimr::NBsController
diff --git a/ydb/core/mind/bscontroller/ya.make b/ydb/core/mind/bscontroller/ya.make
@@ -45,6 +45,7 @@ SRCS(
     resources.h
     scheme.h
     scrub.cpp
+    shred.cpp
     select_groups.cpp
     select_groups.h
     self_heal.cpp

diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto
@@ -1434,5 +1434,18 @@ message TEvControllerValidateConfigResponse {
     optional string ErrorReason = 5;
 }
 
-message TEvControllerValidationTimeout {
+message TEvControllerShredRequest {
+    optional uint64 Generation = 1; // if set, start a new shred iteration; operation is idempotent
+}
+
+message TEvControllerShredResponse {
+    optional uint64 CurrentGeneration = 1; // current shred iteration
+    optional bool Completed = 2; // is it completed?
+    optional uint32 Progress10k = 3; // progress in range [0, 10000]
+}
+
+message TShredState {
+    optional uint64 Generation = 1;
+    optional bool Completed = 2;
+    optional uint32 Progress10k = 3;
 }
diff --git a/ydb/core/protos/counters_bs_controller.proto b/ydb/core/protos/counters_bs_controller.proto
@@ -276,4 +276,5 @@ enum ETxTypes {
     TXTYPE_UPDATE_GROUP = 27 [(TxTypeOpts) = {Name: "TTxUpdateGroup"}];
     TXTYPE_DELETE_BLOB_DEPOT = 28 [(TxTypeOpts) = {Name: "TTxDeleteBlobDepot"}];
     TXTYPE_COMMIT_CONFIG = 29 [(TxTypeOpts) = {Name: "TTxCommitConfig"}];
+    TXTYPE_UPDATE_SHRED = 30 [(TxTypeOpts) = {Name: "TTxUpdateShred"}];
 }
diff --git a/...tests.TestTabletSchemes.test_tablet_schemes_flat_bs_controller_/flat_bs_controller.schema b/...tests.TestTabletSchemes.test_tablet_schemes_flat_bs_controller_/flat_bs_controller.schema
@@ -125,6 +125,11 @@
                 "ColumnId": 25,
                 "ColumnName": "ConfigVersion",
                 "ColumnType": "Uint32"
+            },
+            {
+                "ColumnId": 26,
+                "ColumnName": "ShredState",
+                "ColumnType": "String"
             }
         ],
         "ColumnsDropped": [],
@@ -154,7 +159,8 @@
                     22,
                     23,
                     24,
-                    25
+                    25,
+                    26
                 ],
                 "RoomID": 0,
                 "Codec": 0,