Library "files" for ydb/public/lib/ydb_cli/dump (#12432)

ydb/library/backup/backup.cpp

@@ -4,6 +4,7 @@
 
 #include <ydb/public/api/protos/ydb_table.pb.h>
 #include <ydb/public/lib/ydb_cli/common/recursive_remove.h>
+#include <ydb/public/lib/ydb_cli/dump/files/files.h>
 #include <ydb/public/lib/ydb_cli/dump/util/util.h>
 #include <ydb/public/lib/yson_value/ydb_yson_value.h>
 #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
@@ -32,12 +33,6 @@
 
 namespace NYdb::NBackup {
 
-static constexpr const char *SCHEME_FILE_NAME = "scheme.pb";
-static constexpr const char *PERMISSIONS_FILE_NAME = "permissions.pb";
-static constexpr const char *INCOMPLETE_DATA_FILE_NAME = "incomplete.csv";
-static constexpr const char *INCOMPLETE_FILE_NAME = "incomplete";
-static constexpr const char *EMPTY_FILE_NAME = "empty_dir";
-
 static constexpr size_t IO_BUFFER_SIZE = 2 << 20; // 2 MiB
 static constexpr i64 FILE_SPLIT_THRESHOLD = 128 << 20; // 128 MiB
 static constexpr i64 READ_TABLE_RETRIES = 100;
@@ -284,7 +279,7 @@ TMaybe<TValue> TryReadTable(TDriver driver, const NTable::TTableDescription& des
         VerifyStatus(resultSetStreamPart, TStringBuilder() << "Read next part of " << fullTablePath.Quote() << " failed");
         TResultSet resultSetCurrent = resultSetStreamPart.ExtractPart();
 
-        auto tmpFile = TFile(folderPath.Child(INCOMPLETE_DATA_FILE_NAME), CreateAlways | WrOnly);
+        auto tmpFile = TFile(folderPath.Child(NDump::NFiles::IncompleteData().FileName), CreateAlways | WrOnly);
         TStringStream ss;
         ss.Reserve(IO_BUFFER_SIZE);
 
@@ -322,7 +317,7 @@ TMaybe<TValue> TryReadTable(TDriver driver, const NTable::TTableDescription& des
             }
             if (tmpFile.GetLength() > FILE_SPLIT_THRESHOLD) {
                 CloseAndRename(tmpFile, folderPath.Child(CreateDataFileName((*fileCounter)++)));
-                tmpFile = TFile(folderPath.Child(INCOMPLETE_DATA_FILE_NAME), CreateAlways | WrOnly);
+                tmpFile = TFile(folderPath.Child(NDump::NFiles::IncompleteData().FileName), CreateAlways | WrOnly);
             }
         }
         Flush(tmpFile, ss, lastWrittenPK, lastReadPK);
@@ -465,9 +460,9 @@ void BackupPermissions(TDriver driver, const TString& dbPrefix, const TString& p
 
     TString permissionsStr;
     google::protobuf::TextFormat::PrintToString(proto, &permissionsStr);
-    LOG_D("Write ACL into " << folderPath.Child(PERMISSIONS_FILE_NAME).GetPath().Quote());
+    LOG_D("Write ACL into " << folderPath.Child(NDump::NFiles::Permissions().FileName).GetPath().Quote());
 
-    TFile outFile(folderPath.Child(PERMISSIONS_FILE_NAME), CreateAlways | WrOnly);
+    TFile outFile(folderPath.Child(NDump::NFiles::Permissions().FileName), CreateAlways | WrOnly);
     outFile.Write(permissionsStr.data(), permissionsStr.size());
 }
 
@@ -485,8 +480,8 @@ void BackupTable(TDriver driver, const TString& dbPrefix, const TString& backupP
 
     TString schemaStr;
     google::protobuf::TextFormat::PrintToString(proto, &schemaStr);
-    LOG_D("Write scheme into " << folderPath.Child(SCHEME_FILE_NAME).GetPath().Quote());
-    TFile outFile(folderPath.Child(SCHEME_FILE_NAME), CreateAlways | WrOnly);
+    LOG_D("Write scheme into " << folderPath.Child(NDump::NFiles::TableScheme().FileName).GetPath().Quote());
+    TFile outFile(folderPath.Child(NDump::NFiles::TableScheme().FileName), CreateAlways | WrOnly);
     outFile.Write(schemaStr.data(), schemaStr.size());
 
     BackupPermissions(driver, dbPrefix, path, folderPath);
@@ -535,15 +530,15 @@ static bool IsExcluded(const TString& path, const TVector<TRegExMatch>& exclusio
 static void MaybeCreateEmptyFile(const TFsPath& folderPath) {
     TVector<TString> children;
     folderPath.ListNames(children);
-    if (children.empty() || (children.size() == 1 && children[0] == INCOMPLETE_FILE_NAME)) {
-        TFile(folderPath.Child(EMPTY_FILE_NAME), CreateAlways);
+    if (children.empty() || (children.size() == 1 && children[0] == NDump::NFiles::Incomplete().FileName)) {
+        TFile(folderPath.Child(NDump::NFiles::Empty().FileName), CreateAlways);
     }
 }
 
 void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& backupPrefix,
         const TFsPath folderPath, const TVector<TRegExMatch>& exclusionPatterns,
         bool schemaOnly, bool useConsistentCopyTable, bool avoidCopy, bool preservePoolKinds, bool ordered) {
-    TFile(folderPath.Child(INCOMPLETE_FILE_NAME), CreateAlways);
+    TFile(folderPath.Child(NDump::NFiles::Incomplete().FileName), CreateAlways);
 
     TMap<TString, TAsyncStatus> copiedTablesStatuses;
     TVector<NTable::TCopyItem> tablesToCopy;
@@ -560,12 +555,12 @@ void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& ba
             TFsPath childFolderPath = folderPath.Child(dbIt.GetRelPath());
             LOG_D("Process " << childFolderPath.GetPath().Quote());
             childFolderPath.MkDir();
-            TFile(childFolderPath.Child(INCOMPLETE_FILE_NAME), CreateAlways).Close();
+            TFile(childFolderPath.Child(NDump::NFiles::Incomplete().FileName), CreateAlways).Close();
             if (schemaOnly) {
                 if (dbIt.IsTable()) {
                     BackupTable(driver, dbIt.GetTraverseRoot(), backupPrefix, dbIt.GetRelPath(),
                             childFolderPath, schemaOnly, preservePoolKinds, ordered);
-                    childFolderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists();
+                    childFolderPath.Child(NDump::NFiles::Incomplete().FileName).DeleteIfExists();
                 }
             } else if (!avoidCopy) {
                 if (dbIt.IsTable()) {
@@ -597,16 +592,16 @@ void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& ba
             if (dbIt.IsTable()) {
                 // If table backup was not successful exception should be thrown,
                 // so control flow can't reach this line. Check it just to be sure
-                Y_ENSURE(!childFolderPath.Child(INCOMPLETE_FILE_NAME).Exists());
+                Y_ENSURE(!childFolderPath.Child(NDump::NFiles::Incomplete().FileName).Exists());
             } else if (dbIt.IsDir()) {
                 MaybeCreateEmptyFile(childFolderPath);
                 BackupPermissions(driver, dbIt.GetTraverseRoot(), dbIt.GetRelPath(), childFolderPath);
             }
 
-            childFolderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists();
+            childFolderPath.Child(NDump::NFiles::Incomplete().FileName).DeleteIfExists();
             dbIt.Next();
         }
-        folderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists();
+        folderPath.Child(NDump::NFiles::Incomplete().FileName).DeleteIfExists();
         return;
     }
 
@@ -646,13 +641,13 @@ void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& ba
                 }
             }
 
-            childFolderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists();
+            childFolderPath.Child(NDump::NFiles::Incomplete().FileName).DeleteIfExists();
             dbIt.Next();
         }
     }
     Y_ENSURE(copiedTablesStatuses.empty(), "Some tables was copied but not backuped, example of such table, path# "
             << copiedTablesStatuses.begin()->first.Quote());
-    folderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists();
+    folderPath.Child(NDump::NFiles::Incomplete().FileName).DeleteIfExists();
 }
 
 void CheckedCreateBackupFolder(const TFsPath& folderPath) {

ydb/library/backup/ya.make

@@ -10,6 +10,7 @@ PEERDIR(
     ydb/public/lib/ydb_cli/common
     ydb/public/lib/ydb_cli/dump/util
     ydb/public/lib/yson_value
+    ydb/public/lib/ydb_cli/dump/files
     ydb/public/sdk/cpp/client/ydb_driver
     ydb/public/sdk/cpp/client/ydb_result
     ydb/public/sdk/cpp/client/ydb_table

ydb/public/lib/ydb_cli/commands/ya.make

@@ -51,6 +51,7 @@ PEERDIR(
     ydb/public/lib/ydb_cli/commands/ydb_discovery
     ydb/public/lib/ydb_cli/commands/sdk_core_access
     ydb/public/lib/ydb_cli/dump
+    ydb/public/lib/ydb_cli/dump/files
     ydb/public/lib/ydb_cli/import
     ydb/public/lib/ydb_cli/topic
     ydb/public/sdk/cpp/client/draft

ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp

@@ -5,6 +5,7 @@
 #include <ydb/public/lib/ydb_cli/common/normalize_path.h>
 #include <ydb/public/lib/ydb_cli/common/print_operation.h>
 #include <ydb/public/lib/ydb_cli/common/interactive.h>
+#include <ydb/public/lib/ydb_cli/dump/files/files.h>
 #include <ydb/public/lib/ydb_cli/import/import.h>
 #include <ydb/library/backup/util.h>
 
@@ -18,10 +19,6 @@
 #include <unistd.h>
 #endif
 
-namespace NYdb::NDump {
-    extern const char SCHEME_FILE_NAME[];
-}
-
 namespace NYdb::NConsoleClient {
 
 TCommandImport::TCommandImport()
@@ -172,7 +169,7 @@ int TCommandImportFromS3::Run(TConfig& config) {
                 auto listResult = s3Client->ListObjectKeys(item.Source, token);
                 token = listResult.NextToken;
                 for (TStringBuf key : listResult.Keys) {
-                    if (key.ChopSuffix(NDump::SCHEME_FILE_NAME)) {
+                    if (key.ChopSuffix(NDump::NFiles::TableScheme().FileName)) {
                         TString destination = item.Destination + key.substr(item.Source.size());
                         settings.AppendItem({TString(key), std::move(destination)});
                     }

ydb/public/lib/ydb_cli/dump/dump.cpp

@@ -11,11 +11,6 @@
 namespace NYdb {
 namespace NDump {
 
-extern const char SCHEME_FILE_NAME[] = "scheme.pb";
-extern const char PERMISSIONS_FILE_NAME[] = "permissions.pb";
-extern const char INCOMPLETE_FILE_NAME[] = "incomplete";
-extern const char EMPTY_FILE_NAME[] = "empty_dir";
-
 TString DataFileName(ui32 id) {
     return Sprintf("data_%02d.csv", id);
 }

ydb/public/lib/ydb_cli/dump/dump.h

@@ -16,11 +16,6 @@ class TDriver;
 
 namespace NDump {
 
-extern const char SCHEME_FILE_NAME[10];
-extern const char PERMISSIONS_FILE_NAME[15];
-extern const char INCOMPLETE_FILE_NAME[11];
-extern const char EMPTY_FILE_NAME[10];
-
 TString DataFileName(ui32 id);
 
 /// dump

ydb/public/lib/ydb_cli/dump/files/files.cpp

@@ -0,0 +1,53 @@
+#include "files.h"
+
+namespace NYdb::NDump::NFiles {
+
+enum EFilesType {
+    SCHEME = 0,
+    PERMISSIONS,
+    CHANGEFEED_DESCRIPTION,
+    TOPIC_DESCRIPTION,
+    INCOMPLETE_DATA,
+    INCOMPLETE,
+    EMPTY,
+};
+
+static constexpr TFileInfo FILES_INFO[] = {
+    {"scheme.pb", "scheme"},
+    {"permissions.pb", "ACL"},
+    {"changefeed_description.pb", "changefeed"},
+    {"topic_description.pb", "topic"},
+    {"incomplete.csv", "incomplete"},
+    {"incomplete", "incomplete"},
+    {"empty_dir", "empty_dir"},
+};
+
+const TFileInfo& TableScheme() {
+    return FILES_INFO[SCHEME];
+}
+
+const TFileInfo& Permissions() {
+    return FILES_INFO[PERMISSIONS];
+}
+
+const TFileInfo& Changefeed() {
+    return FILES_INFO[CHANGEFEED_DESCRIPTION];
+}
+
+const TFileInfo& Topic() {
+    return FILES_INFO[TOPIC_DESCRIPTION];
+}
+
+const TFileInfo& IncompleteData() {
+    return FILES_INFO[INCOMPLETE_DATA];
+}
+
+const TFileInfo& Incomplete() {
+    return FILES_INFO[INCOMPLETE];
+}
+
+const TFileInfo& Empty() {
+    return FILES_INFO[EMPTY];
+}
+
+} // NYdb::NDump::NFiles

ydb/public/lib/ydb_cli/dump/files/files.h

@@ -0,0 +1,18 @@
+#pragma once
+
+namespace NYdb::NDump::NFiles {
+
+struct TFileInfo {
+    const char* FileName;
+    const char* LogObjectType;
+};
+
+const TFileInfo& TableScheme();
+const TFileInfo& Permissions();
+const TFileInfo& Changefeed();
+const TFileInfo& Topic();
+const TFileInfo& IncompleteData();
+const TFileInfo& Incomplete();
+const TFileInfo& Empty();
+
+} // NYdb::NDump:NFiles

ydb/public/lib/ydb_cli/dump/files/ya.make

@@ -0,0 +1,7 @@
+LIBRARY()
+
+SRCS(
+    files.cpp
+)
+
+END()

ydb/public/lib/ydb_cli/dump/restore_impl.cpp

@@ -7,6 +7,7 @@
 #include <ydb/public/lib/ydb_cli/common/recursive_list.h>
 #include <ydb/public/lib/ydb_cli/common/recursive_remove.h>
 #include <ydb/public/lib/ydb_cli/common/retry_func.h>
+#include <ydb/public/lib/ydb_cli/dump/files/files.h>
 #include <ydb/public/lib/ydb_cli/dump/util/log.h>
 #include <ydb/public/lib/ydb_cli/dump/util/util.h>
 
@@ -275,16 +276,16 @@ TRestoreResult TRestoreClient::RestoreFolder(const TFsPath& fsPath, const TStrin
             TStringBuilder() << "Specified folder is not a directory: " << fsPath.GetPath());
     }
 
-    if (IsFileExists(fsPath.Child(INCOMPLETE_FILE_NAME))) {
+    if (IsFileExists(fsPath.Child(NFiles::Incomplete().FileName))) {
         return Result<TRestoreResult>(EStatus::BAD_REQUEST,
             TStringBuilder() << "There is incomplete file in folder: " << fsPath.GetPath());
     }
 
-    if (IsFileExists(fsPath.Child(SCHEME_FILE_NAME))) {
+    if (IsFileExists(fsPath.Child(NFiles::TableScheme().FileName))) {
         return RestoreTable(fsPath, Join('/', dbPath, fsPath.GetName()), settings, oldEntries);
     }
 
-    if (IsFileExists(fsPath.Child(EMPTY_FILE_NAME))) {
+    if (IsFileExists(fsPath.Child(NFiles::Empty().FileName))) {
         return RestoreEmptyDir(fsPath, Join('/', dbPath, fsPath.GetName()), settings, oldEntries);
     }
 
@@ -293,9 +294,9 @@ TRestoreResult TRestoreClient::RestoreFolder(const TFsPath& fsPath, const TStrin
     TVector<TFsPath> children;
     fsPath.List(children);
     for (const auto& child : children) {
-        if (IsFileExists(child.Child(SCHEME_FILE_NAME))) {
+        if (IsFileExists(child.Child(NFiles::TableScheme().FileName))) {
             result = RestoreTable(child, Join('/', dbPath, child.GetName()), settings, oldEntries);
-        } else if (IsFileExists(child.Child(EMPTY_FILE_NAME))) {
+        } else if (IsFileExists(child.Child(NFiles::Empty().FileName))) {
             result = RestoreEmptyDir(child, Join('/', dbPath, child.GetName()), settings, oldEntries);
         } else if (child.IsDirectory()) {
             result = RestoreFolder(child, Join('/', dbPath, child.GetName()), settings, oldEntries);
@@ -314,12 +315,12 @@ TRestoreResult TRestoreClient::RestoreTable(const TFsPath& fsPath, const TString
 {
     LOG_D("Process " << fsPath.GetPath().Quote());
 
-    if (fsPath.Child(INCOMPLETE_FILE_NAME).Exists()) {
+    if (fsPath.Child(NFiles::Incomplete().FileName).Exists()) {
         return Result<TRestoreResult>(EStatus::BAD_REQUEST,
             TStringBuilder() << "There is incomplete file in folder: " << fsPath.GetPath());
     }
 
-    auto scheme = ReadTableScheme(fsPath.Child(SCHEME_FILE_NAME), Log.get());
+    auto scheme = ReadTableScheme(fsPath.Child(NFiles::TableScheme().FileName), Log.get());
     auto dumpedDesc = TableDescriptionFromProto(scheme);
 
     if (dumpedDesc.GetAttributes().contains(DOC_API_TABLE_VERSION_ATTR) && settings.SkipDocumentTables_) {
@@ -629,7 +630,7 @@ TRestoreResult TRestoreClient::RestoreIndexes(const TString& dbPath, const TTabl
 TRestoreResult TRestoreClient::RestorePermissions(const TFsPath& fsPath, const TString& dbPath,
     const TRestoreSettings& settings, const THashSet<TString>& oldEntries)
 {   
-    if (fsPath.Child(INCOMPLETE_FILE_NAME).Exists()) {
+    if (fsPath.Child(NFiles::Incomplete().FileName).Exists()) {
         return Result<TRestoreResult>(EStatus::BAD_REQUEST,
             TStringBuilder() << "There is incomplete file in folder: " << fsPath.GetPath());
     }
@@ -642,13 +643,13 @@ TRestoreResult TRestoreClient::RestorePermissions(const TFsPath& fsPath, const T
         return Result<TRestoreResult>();
     }
 
-    if (!fsPath.Child(PERMISSIONS_FILE_NAME).Exists()) {
+    if (!fsPath.Child(NFiles::Permissions().FileName).Exists()) {
         return Result<TRestoreResult>();
     }
 
     LOG_D("Restore ACL " << fsPath.GetPath().Quote() << " to " << dbPath.Quote());
 
-    auto permissions = ReadPermissions(fsPath.Child(PERMISSIONS_FILE_NAME), Log.get());
+    auto permissions = ReadPermissions(fsPath.Child(NFiles::Permissions().FileName), Log.get());
     return ModifyPermissions(SchemeClient, dbPath, TModifyPermissionsSettings(permissions));
 }
 
@@ -657,7 +658,7 @@ TRestoreResult TRestoreClient::RestoreEmptyDir(const TFsPath& fsPath, const TStr
 {
     LOG_D("Process " << fsPath.GetPath().Quote());
 
-    if (fsPath.Child(INCOMPLETE_FILE_NAME).Exists()) {
+    if (fsPath.Child(NFiles::Incomplete().FileName).Exists()) {
         return Result<TRestoreResult>(EStatus::BAD_REQUEST,
             TStringBuilder() << "There is incomplete file in folder: " << fsPath.GetPath());
     }

ydb/public/lib/ydb_cli/dump/ya.make

@@ -16,6 +16,7 @@ PEERDIR(
     ydb/library/backup
     ydb/public/api/protos
     ydb/public/lib/ydb_cli/common
+    ydb/public/lib/ydb_cli/dump/files
     ydb/public/lib/ydb_cli/dump/util
     ydb/public/sdk/cpp/client/ydb_proto
 )