Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dont use removed portions before remove from local_db - prevent race … #12383

Merged
merged 1 commit into from
Dec 9, 2024
+24 −22
Merged

dont use removed portions before remove from local_db - prevent race … #12383

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 7 additions & 9 deletions ydb/core/tx/columnshard/columnshard_impl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1334,7 +1334,9 @@ class TPortionConstructorV2 {
}

NOlap::TPortionDataAccessor BuildAccessor() {
AFL_VERIFY(PortionInfo && Records && Indexes);
AFL_VERIFY(PortionInfo);
AFL_VERIFY(Records)("portion_id", PortionInfo->GetPortionId())("path_id", PortionInfo->GetPathId());
AFL_VERIFY(Indexes)("portion_id", PortionInfo->GetPortionId())("path_id", PortionInfo->GetPathId());
std::vector<NOlap::TColumnChunkLoadContextV1> records = Records->BuildRecordsV1();
return NOlap::TPortionAccessorConstructor::BuildForLoading(std::move(PortionInfo), std::move(records), std::move(*Indexes));
}
Expand Down Expand Up @@ -1418,17 +1420,13 @@ class TTxAskPortionChunks: public TTransactionBase<TColumnShard> {
auto p = i.second.back();
TPortionConstructorV2 constructor(p);
{
auto rowset = db.Table<NColumnShard::Schema::IndexColumnsV2>().Prefix(p->GetPathId(), p->GetPortionId()).Select();
auto rowset = db.Table<NColumnShard::Schema::IndexColumnsV2>().Key(p->GetPathId(), p->GetPortionId()).Select();
if (!rowset.IsReady()) {
return false;
}
while (!rowset.EndOfSet()) {
NOlap::TColumnChunkLoadContextV2 info(rowset);
constructor.SetRecords(std::move(info));
if (!rowset.Next()) {
return false;
}
}
AFL_VERIFY(!rowset.EndOfSet())("path_id", p->GetPathId())("portion_id", p->GetPortionId())("debug", p->DebugString(true));
NOlap::TColumnChunkLoadContextV2 info(rowset);
constructor.SetRecords(std::move(info));
}
std::vector<NOlap::TIndexChunkLoadContext> indexes;
if (p->GetSchema(Self->GetIndexAs<NOlap::TColumnEngineForLogs>().GetVersionedIndex())->GetIndexesCount()) {
Expand Down
4 changes: 2 additions & 2 deletions ydb/core/tx/columnshard/engines/reader/sys_view/abstract/filler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ NKikimr::TConclusionStatus TMetadataFromStore::DoFillMetadata(const NColumnShard
auto pathInfos = logsIndex->GetTables(fromPathId, toPathId);
for (auto&& pathInfo : pathInfos) {
if (pathIds.emplace(pathInfo->GetPathId()).second) {
metadata->IndexGranules.emplace_back(BuildGranuleView(*pathInfo, metadata->IsDescSorted()));
metadata->IndexGranules.emplace_back(BuildGranuleView(*pathInfo, metadata->IsDescSorted(), metadata->GetRequestSnapshot()));
}
}
}
Expand Down Expand Up @@ -52,7 +52,7 @@ NKikimr::TConclusionStatus TMetadataFromTable::DoFillMetadata(const NColumnShard
if (!pathInfo) {
continue;
}
metadata->IndexGranules.emplace_back(BuildGranuleView(*pathInfo, metadata->IsDescSorted()));
metadata->IndexGranules.emplace_back(BuildGranuleView(*pathInfo, metadata->IsDescSorted(), metadata->GetRequestSnapshot()));
break;
}
}
Expand Down
8 changes: 4 additions & 4 deletions ydb/core/tx/columnshard/engines/reader/sys_view/abstract/filler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ class IMetadataFiller {
private:
virtual TConclusionStatus DoFillMetadata(const NColumnShard::TColumnShard* shard, const std::shared_ptr<TReadMetadataBase>& metadata, const TReadDescription& read) const = 0;

virtual NAbstract::TGranuleMetaView DoBuildGranuleView(const TGranuleMeta& granule, const bool reverse) const {
return NAbstract::TGranuleMetaView(granule, reverse);
virtual NAbstract::TGranuleMetaView DoBuildGranuleView(const TGranuleMeta& granule, const bool reverse, const TSnapshot& reqSnapshot) const {
return NAbstract::TGranuleMetaView(granule, reverse, reqSnapshot);
}
public:
virtual ~IMetadataFiller() = default;
Expand All @@ -19,8 +19,8 @@ class IMetadataFiller {
return DoFillMetadata(shard, metadata, read);
}

NAbstract::TGranuleMetaView BuildGranuleView(const TGranuleMeta& granule, const bool reverse) const {
return DoBuildGranuleView(granule, reverse);
NAbstract::TGranuleMetaView BuildGranuleView(const TGranuleMeta& granule, const bool reverse, const TSnapshot& reqSnapshot) const {
return DoBuildGranuleView(granule, reverse, reqSnapshot);
}

};
Expand Down
5 changes: 4 additions & 1 deletion ydb/core/tx/columnshard/engines/reader/sys_view/abstract/granule_view.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,13 @@ class TGranuleMetaView {
YDB_READONLY_DEF(TPortions, Portions);
YDB_READONLY_DEF(std::vector<NStorageOptimizer::TTaskDescription>, OptimizerTasks);
public:
TGranuleMetaView(const TGranuleMeta& granule, const bool reverse)
TGranuleMetaView(const TGranuleMeta& granule, const bool reverse, const TSnapshot& reqSnapshot)
: PathId(granule.GetPathId())
{
for (auto&& i : granule.GetPortions()) {
if (i.second->IsRemovedFor(reqSnapshot)) {
continue;
}
Portions.emplace_back(i.second);
}

Expand Down
2 changes: 1 addition & 1 deletion ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ std::vector<std::pair<TString, NKikimr::NScheme::TTypeInfo>> TReadStatsMetadata:
return GetColumns(TStatsIterator::StatsSchema, TStatsIterator::StatsSchema.KeyColumns);
}

std::shared_ptr<NKikimr::NOlap::NReader::NSysView::NAbstract::TReadStatsMetadata> TConstructor::BuildMetadata(
std::shared_ptr<NAbstract::TReadStatsMetadata> TConstructor::BuildMetadata(
const NColumnShard::TColumnShard* self, const TReadDescription& read) const {
auto* index = self->GetIndexOptional();
return std::make_shared<TReadStatsMetadata>(index ? index->CopyVersionedIndexPtr() : nullptr, self->TabletID(),
Expand Down
2 changes: 1 addition & 1 deletion ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ class TReadStatsMetadata: public NAbstract::TReadStatsMetadata {
public:
using TBase::TBase;

virtual std::unique_ptr<TScanIteratorBase> StartScan(const std::shared_ptr<TReadContext>& /*readContext*/) const override;
virtual std::unique_ptr<TScanIteratorBase> StartScan(const std::shared_ptr<TReadContext>& readContext) const override;
virtual std::vector<std::pair<TString, NScheme::TTypeInfo>> GetKeyYqlSchema() const override;
};

Expand Down
9 changes: 5 additions & 4 deletions ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ class TStatsIterator : public NAbstract::TStatsIterator<NKikimr::NSysView::Schem

class TMetadataFromStore: public NAbstract::TMetadataFromStore {
protected:
virtual NAbstract::TGranuleMetaView DoBuildGranuleView(const TGranuleMeta& granule, const bool reverse) const override {
NAbstract::TGranuleMetaView result(granule, reverse);
virtual NAbstract::TGranuleMetaView DoBuildGranuleView(const TGranuleMeta& granule, const bool reverse, const TSnapshot& reqSnapshot) const override {
NAbstract::TGranuleMetaView result(granule, reverse, reqSnapshot);
result.FillOptimizerTasks(granule, reverse);
return result;
}
Expand All @@ -52,8 +52,9 @@ class TMetadataFromStore: public NAbstract::TMetadataFromStore {

class TMetadataFromTable: public NAbstract::TMetadataFromTable {
protected:
virtual NAbstract::TGranuleMetaView DoBuildGranuleView(const TGranuleMeta& granule, const bool reverse) const override {
NAbstract::TGranuleMetaView result(granule, reverse);
virtual NAbstract::TGranuleMetaView DoBuildGranuleView(
const TGranuleMeta& granule, const bool reverse, const TSnapshot& reqSnapshot) const override {
NAbstract::TGranuleMetaView result(granule, reverse, reqSnapshot);
result.FillOptimizerTasks(granule, reverse);
return result;
}
Expand Down
Loading