Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PageCtl: Support getting blob data from PageStorage instance (release-7.5) #9717

Merged
merged 2 commits into from
Dec 11, 2024
Merged

PageCtl: Support getting blob data from PageStorage instance (release-7.5) #9717

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f60e394
Support getting blob data from PageStorage instance
JaySon-Huang Dec 11, 2024
e743205
Fix type
JaySon-Huang Dec 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dbms/src/Storages/Page/V3/PageEntry.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ struct fmt::formatter<DB::PS::V3::PageEntryV3>

return format_to(
ctx.out(),
"PageEntry{{file: {}, offset: 0x{:X}, size: {}, checksum: 0x{:X}, tag: {}, field_offsets: [{}], "
"PageEntry{{file: {}, offset: {}, size: {}, checksum: 0x{:X}, tag: {}, field_offsets: [{}], "
"checkpoint_info: {}}}",
entry.file_id,
entry.offset,
Expand Down
119 changes: 87 additions & 32 deletions dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@
#include <Poco/ConsoleChannel.h>
#include <Poco/PatternFormatter.h>
#include <Server/CLIService.h>
#include <Storages/Page/PageDefinesBase.h>
#include <Storages/Page/V3/PageDefines.h>
#include <Storages/Page/V3/PageDirectory.h>
#include <Storages/Page/V3/PageDirectoryFactory.h>
#include <Storages/Page/V3/PageStorageImpl.h>
#include <Storages/Page/V3/Universal/RaftDataReader.h>
#include <Storages/Page/V3/Universal/UniversalPageId.h>
#include <Storages/Page/V3/Universal/UniversalPageIdFormatImpl.h>
#include <Storages/Page/V3/Universal/UniversalPageStorage.h>
#include <Storages/PathPool.h>
Expand All @@ -29,6 +32,7 @@
#include <common/types.h>

#include <boost/program_options.hpp>
#include <cstdint>
#include <magic_enum.hpp>
#include <unordered_set>

Expand All @@ -47,12 +51,15 @@ struct ControlOptions
CHECK_ALL_DATA_CRC = 4,
DISPLAY_WAL_ENTRIES = 5,
DISPLAY_REGION_INFO = 6,
DISPLAY_BLOB_DATA = 7,
};

std::vector<std::string> paths;
DisplayType mode = DisplayType::DISPLAY_SUMMARY_INFO;
UInt64 page_id = UINT64_MAX;
UInt32 blob_id = UINT32_MAX;
BlobFileId blob_id = INVALID_BLOBFILE_ID;
BlobFileOffset blob_offset = INVALID_BLOBFILE_OFFSET;
size_t blob_size = UINT64_MAX;
UInt64 namespace_id = DB::TEST_NAMESPACE_ID;
StorageType storage_type = StorageType::Unknown; // only useful for universal page storage
UInt32 keyspace_id = NullspaceID; // only useful for universal page storage
Expand Down Expand Up @@ -85,6 +92,7 @@ ControlOptions ControlOptions::parse(int argc, char ** argv)
4 is check every data is valid
5 is dump entries in WAL log files
6 is display all region info
7 is display blob data (in hex)
)") //
("show_entries",
value<bool>()->default_value(true),
Expand All @@ -106,8 +114,14 @@ ControlOptions ControlOptions::parse(int argc, char ** argv)
value<UInt64>()->default_value(UINT64_MAX),
"Query a single Page id, and print its version chain.") //
("blob_id,B",
value<UInt32>()->default_value(UINT32_MAX),
"Query a single Blob id, and print its data distribution.") //
value<BlobFileId>()->default_value(INVALID_BLOBFILE_ID),
"Specify the blob_id") //
("blob_offset",
value<BlobFileOffset>()->default_value(INVALID_BLOBFILE_OFFSET),
"Specify the offset.") //
("blob_size",
value<size_t>()->default_value(0),
"Specify the size.") //
//
("imitative,I",
value<bool>()->default_value(true),
Expand Down Expand Up @@ -140,7 +154,9 @@ ControlOptions ControlOptions::parse(int argc, char ** argv)
opt.paths = options["paths"].as<std::vector<std::string>>();
auto mode_int = options["mode"].as<int>();
opt.page_id = options["page_id"].as<UInt64>();
opt.blob_id = options["blob_id"].as<UInt32>();
opt.blob_id = options["blob_id"].as<BlobFileId>();
opt.blob_offset = options["blob_offset"].as<BlobFileOffset>();
opt.blob_size = options["blob_size"].as<size_t>();
opt.show_entries = options["show_entries"].as<bool>();
opt.check_fields = options["check_fields"].as<bool>();
auto storage_type_int = options["storage_type"].as<int>();
Expand Down Expand Up @@ -346,6 +362,12 @@ class PageStorageControlV3
}
break;
}
case ControlOptions::DisplayType::DISPLAY_BLOB_DATA:
{
String hex_data = getBlobData(blob_store, opts.blob_id, opts.blob_offset, opts.blob_size);
fmt::print("hex:{}\n", hex_data);
break;
}
default:
std::cout << "Invalid display mode." << std::endl;
break;
Expand All @@ -372,7 +394,7 @@ class PageStorageControlV3
return 0;
}

static String getBlobsInfo(typename Trait::BlobStore & blob_store, UInt32 blob_id)
static String getBlobsInfo(typename Trait::BlobStore & blob_store, BlobFileId blob_id)
{
auto stat_info = [](const BlobStats::BlobStatPtr & stat, const String & path) {
FmtBuffer stat_str;
Expand Down Expand Up @@ -402,7 +424,7 @@ class PageStorageControlV3
{
for (const auto & stat : stats)
{
if (blob_id != UINT32_MAX)
if (blob_id != INVALID_BLOBFILE_ID)
{
if (stat->id == blob_id)
{
Expand All @@ -416,7 +438,7 @@ class PageStorageControlV3
}
}

if (blob_id != UINT32_MAX)
if (blob_id != INVALID_BLOBFILE_ID)
{
stats_info.fmtAppend(" no found blob {}", blob_id);
}
Expand Down Expand Up @@ -447,34 +469,41 @@ class PageStorageControlV3
size_t count = 0;
for (const auto & [version, entry_or_del] : versioned_entries->entries)
{
const auto & entry = entry_or_del.entry;
page_str.fmtAppend(
" entry {}\n"
" sequence: {}\n"
" epoch: {}\n"
" is del: {}\n"
" blob id: {}\n"
" offset: {}\n"
" size: {}\n"
" crc: 0x{:X}\n", //
count++, //
version.sequence, //
version.epoch, //
entry_or_del.isDelete(), //
entry.file_id, //
entry.offset, //
entry.size, //
entry.checksum, //
entry.field_offsets.size() //
);
if (!entry.field_offsets.empty())
if (entry_or_del.isEntry())
{
page_str.append(" field offset:\n");
for (const auto & [offset, crc] : entry.field_offsets)
const auto & entry = entry_or_del.entry;
page_str.fmtAppend(
" entry {}\n"
" sequence: {}\n"
" epoch: {}\n"
" is del: false\n"
" blob id: {}\n"
" offset: {}\n"
" size: {}\n"
" crc: 0x{:X}\n", //
count++, //
version.sequence, //
version.epoch, //
entry.file_id, //
entry.offset, //
entry.size, //
entry.checksum, //
entry.field_offsets.size() //
);
if (!entry.field_offsets.empty())
{
page_str.fmtAppend(" offset: {} crc: 0x{:X}\n", offset, crc);
page_str.append(" field offset:\n");
for (const auto & [offset, crc] : entry.field_offsets)
{
page_str.fmtAppend(" offset: {} crc: 0x{:X}\n", offset, crc);
}
page_str.append("\n");
}
page_str.append("\n");
}
else
{
page_str.append(" entry is null\n"
" is del: true\n");
}
}
return page_str.toString();
Expand Down Expand Up @@ -814,6 +843,32 @@ class PageStorageControlV3
return error_msg.toString();
}

static String getBlobData(
typename Trait::BlobStore & blob_store,
BlobFileId blob_id,
BlobFileOffset offset,
size_t size)
{
auto page_id = []() {
if constexpr (std::is_same_v<Trait, u128::PageStorageControlV3Trait>)
return PageIdV3Internal(0, 0);
else
return UniversalPageId("");
}();
char * buffer = new char[size];
blob_store.read(page_id, blob_id, offset, buffer, size, nullptr, false);

using ChecksumClass = Digest::CRC64;
ChecksumClass digest;
digest.update(buffer, size);
auto checksum = digest.checksum();
fmt::print("checksum: 0x{:X}\n", checksum);

auto hex_str = Redact::keyToHexString(buffer, size);
delete[] buffer;
return hex_str;
}

private:
ControlOptions options;
};
Expand Down
3 changes: 3 additions & 0 deletions release-centos7-llvm/env/.gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
tiflash-env
tiflash-env-*.tar.xz
llvm-project
sysroot
*.log