Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

reuse code for portion meta #13065

Merged
merged 1 commit into from
Dec 28, 2024
Merged

reuse code for portion meta #13065

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ TPortionMeta TPortionMetaConstructor::Build() {
if (TierName) {
result.TierName = *TierName;
}
AFL_VERIFY(BlobIds.size());
TBase::FullValidation();
result.BlobIds = BlobIds;
result.BlobIds.shrink_to_fit();
result.CompactionLevel = *TValidator::CheckNotNull(CompactionLevel);
Expand Down
32 changes: 2 additions & 30 deletions ydb/core/tx/columnshard/engines/portions/constructor_meta.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@ namespace NKikimr::NOlap {
class TPortionInfoConstructor;
struct TIndexInfo;

class TPortionMetaConstructor {
class TPortionMetaConstructor: public TPortionMetaBase {
private:
using TBase = TPortionMetaBase;
std::optional<NArrow::TFirstLastSpecialKeys> FirstAndLastPK;
std::optional<TString> TierName;
std::optional<TSnapshot> RecordSnapshotMin;
Expand All @@ -27,8 +28,6 @@ class TPortionMetaConstructor {

std::optional<ui32> DeletionsCount;

std::vector<TUnifiedBlobId> BlobIds;

friend class TPortionInfoConstructor;
friend class TPortionAccessorConstructor;
void FillMetaInfo(const NArrow::TFirstLastSpecialKeys& primaryKeys, const ui32 deletionsCount,
Expand All @@ -42,15 +41,6 @@ class TPortionMetaConstructor {
return linkRange.RestoreRange(GetBlobId(linkRange.GetBlobIdxVerified()));
}

ui32 GetBlobIdsCount() const {
return BlobIds.size();
}

const TUnifiedBlobId& GetBlobId(const TBlobRangeLink16::TLinkId linkId) const {
AFL_VERIFY(linkId < BlobIds.size());
return BlobIds[linkId];
}

TBlobRangeLink16::TLinkId RegisterBlobId(const TUnifiedBlobId& blobId) {
AFL_VERIFY(blobId.IsValid());
TBlobRangeLink16::TLinkId idx = 0;
Expand All @@ -64,24 +54,6 @@ class TPortionMetaConstructor {
return idx;
}

std::optional<TBlobRangeLink16::TLinkId> GetBlobIdxOptional(const TUnifiedBlobId& blobId) const {
AFL_VERIFY(blobId.IsValid());
TBlobRangeLink16::TLinkId idx = 0;
for (auto&& i : BlobIds) {
if (i == blobId) {
return idx;
}
++idx;
}
return std::nullopt;
}

TBlobRangeLink16::TLinkId GetBlobIdxVerified(const TUnifiedBlobId& blobId) const {
auto result = GetBlobIdxOptional(blobId);
AFL_VERIFY(result);
return *result;
}

void SetCompactionLevel(const ui64 level) {
CompactionLevel = level;
}
Expand Down
2 changes: 1 addition & 1 deletion ydb/core/tx/columnshard/engines/portions/meta.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ NKikimrTxColumnShard::TIndexPortionMeta TPortionMeta::SerializeToProto() const {

RecordSnapshotMin.SerializeToProto(*portionMeta.MutableRecordSnapshotMin());
RecordSnapshotMax.SerializeToProto(*portionMeta.MutableRecordSnapshotMax());
for (auto&& i : BlobIds) {
for (auto&& i : GetBlobIds()) {
*portionMeta.AddBlobIds() = i.GetLogoBlobId().AsBinaryString();
}
return portionMeta;
Expand Down
85 changes: 51 additions & 34 deletions ydb/core/tx/columnshard/engines/portions/meta.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,56 @@ namespace NKikimr::NOlap {

struct TIndexInfo;

struct TPortionMeta {
class TPortionMetaBase {
protected:
std::vector<TUnifiedBlobId> BlobIds;
public:
const std::vector<TUnifiedBlobId>& GetBlobIds() const {
return BlobIds;
}

const TUnifiedBlobId& GetBlobId(const TBlobRangeLink16::TLinkId linkId) const {
AFL_VERIFY(linkId < GetBlobIds().size());
return BlobIds[linkId];
}

ui32 GetBlobIdsCount() const {
return BlobIds.size();
}

void FullValidation() const {
for (auto&& i : BlobIds) {
AFL_VERIFY(i.BlobSize());
}
AFL_VERIFY(BlobIds.size());
}

std::optional<TBlobRangeLink16::TLinkId> GetBlobIdxOptional(const TUnifiedBlobId& blobId) const {
AFL_VERIFY(blobId.IsValid());
TBlobRangeLink16::TLinkId idx = 0;
for (auto&& i : BlobIds) {
if (i == blobId) {
return idx;
}
++idx;
}
return std::nullopt;
}

ui64 GetMetadataMemorySize() const {
return GetBlobIds().size() * sizeof(TUnifiedBlobId);
}

TBlobRangeLink16::TLinkId GetBlobIdxVerified(const TUnifiedBlobId& blobId) const {
auto result = GetBlobIdxOptional(blobId);
AFL_VERIFY(result);
return *result;
}
};

class TPortionMeta: public TPortionMetaBase {
private:
using TBase = TPortionMetaBase;
NArrow::TFirstLastSpecialKeys ReplaceKeyEdges; // first and last PK rows
YDB_READONLY_DEF(TString, TierName);
YDB_READONLY(ui32, DeletionsCount, 0);
Expand All @@ -25,7 +73,6 @@ struct TPortionMeta {
YDB_READONLY(ui32, ColumnBlobBytes, 0);
YDB_READONLY(ui32, IndexRawBytes, 0);
YDB_READONLY(ui32, IndexBlobBytes, 0);
YDB_READONLY_DEF(std::vector<TUnifiedBlobId>, BlobIds);

friend class TPortionMetaConstructor;
friend class TPortionInfo;
Expand All @@ -41,10 +88,7 @@ struct TPortionMeta {
TSnapshot RecordSnapshotMax;

void FullValidation() const {
for (auto&& i : BlobIds) {
AFL_VERIFY(i.BlobSize());
}
AFL_VERIFY(BlobIds.size());
TBase::FullValidation();
AFL_VERIFY(RecordsCount);
AFL_VERIFY(ColumnRawBytes);
AFL_VERIFY(ColumnBlobBytes);
Expand All @@ -55,37 +99,10 @@ struct TPortionMeta {
return ReplaceKeyEdges;
}

const TUnifiedBlobId& GetBlobId(const TBlobRangeLink16::TLinkId linkId) const {
AFL_VERIFY(linkId < GetBlobIds().size());
return BlobIds[linkId];
}

ui32 GetBlobIdsCount() const {
return BlobIds.size();
}

void ResetCompactionLevel(const ui32 level) {
CompactionLevel = level;
}

std::optional<TBlobRangeLink16::TLinkId> GetBlobIdxOptional(const TUnifiedBlobId& blobId) const {
AFL_VERIFY(blobId.IsValid());
TBlobRangeLink16::TLinkId idx = 0;
for (auto&& i : BlobIds) {
if (i == blobId) {
return idx;
}
++idx;
}
return std::nullopt;
}

TBlobRangeLink16::TLinkId GetBlobIdxVerified(const TUnifiedBlobId& blobId) const {
auto result = GetBlobIdxOptional(blobId);
AFL_VERIFY(result);
return *result;
}

using EProduced = NPortion::EProduced;

NArrow::TReplaceKey IndexKeyStart;
Expand All @@ -96,7 +113,7 @@ struct TPortionMeta {
std::optional<TString> GetTierNameOptional() const;

ui64 GetMetadataMemorySize() const {
return sizeof(TPortionMeta) + ReplaceKeyEdges.GetMemorySize() + BlobIds.size() * sizeof(TUnifiedBlobId);
return sizeof(TPortionMeta) + ReplaceKeyEdges.GetMemorySize() + TBase::GetMetadataMemorySize();
}

NKikimrTxColumnShard::TIndexPortionMeta SerializeToProto() const;
Expand Down
Loading