Skip to content

Commit

Permalink
Merge 7ea8dd3 into 9740792
Browse files Browse the repository at this point in the history
  • Loading branch information
@GrigoriyPA
GrigoriyPA authored Jan 5, 2025
2 parents 9740792 + 7ea8dd3 commit 1195957
Show file tree
Hide file tree
Showing 7 changed files with 143 additions and 90 deletions.
140 changes: 84 additions & 56 deletions ydb/tests/tools/kqprun/kqprun.cpp
View file

Large diffs are not rendered by default.

7 changes: 5 additions & 2 deletions ydb/tests/tools/kqprun/src/actors.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ class TRunScriptActorMock : public NActors::TActorBootstrapped<TRunScriptActorMo
public:
TRunScriptActorMock(TQueryRequest request, NThreading::TPromise<TQueryResponse> promise, TProgressCallback progressCallback)
: TargetNode_(request.TargetNode)
, QueryId_(request.QueryId)
, Request_(std::move(request.Event))
, Promise_(promise)
, ResultRowsLimit_(std::numeric_limits<ui64>::max())
Expand Down Expand Up @@ -83,12 +84,14 @@ class TRunScriptActorMock : public NActors::TActorBootstrapped<TRunScriptActorMo

void Handle(NKikimr::NKqp::TEvKqpExecuter::TEvExecuterProgress::TPtr& ev) {
if (ProgressCallback_) {
ProgressCallback_(ev->Get()->Record);
ProgressCallback_(QueryId_, ev->Get()->Record);
}
}

private:
ui32 TargetNode_ = 0;
const ui32 TargetNode_ = 0;
const size_t QueryId_ = 0;

std::unique_ptr<NKikimr::NKqp::TEvKqp::TEvQueryRequest> Request_;
NThreading::TPromise<TQueryResponse> Promise_;
ui64 ResultRowsLimit_;
Expand Down
3 changes: 2 additions & 1 deletion ydb/tests/tools/kqprun/src/actors.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ struct TQueryRequest {
ui32 TargetNode;
ui64 ResultRowsLimit;
ui64 ResultSizeLimit;
size_t QueryId;
};

struct TCreateSessionRequest {
Expand Down Expand Up @@ -75,7 +76,7 @@ struct TEvPrivate {
};
};

using TProgressCallback = std::function<void(const NKikimrKqp::TEvExecuterProgress&)>;
using TProgressCallback = std::function<void(ui64 queryId, const NKikimrKqp::TEvExecuterProgress& executerProgress)>;

NActors::IActor* CreateRunScriptActorMock(TQueryRequest request, NThreading::TPromise<TQueryResponse> promise, TProgressCallback progressCallback);

Expand Down
18 changes: 14 additions & 4 deletions ydb/tests/tools/kqprun/src/common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,14 +72,15 @@ struct TRunnerOptions {

IOutputStream* ResultOutput = nullptr;
IOutputStream* SchemeQueryAstOutput = nullptr;
IOutputStream* ScriptQueryAstOutput = nullptr;
IOutputStream* ScriptQueryPlanOutput = nullptr;
TString ScriptQueryTimelineFile;
TString InProgressStatisticsOutputFile;
std::vector<IOutputStream*> ScriptQueryAstOutputs;
std::vector<IOutputStream*> ScriptQueryPlanOutputs;
std::vector<TString> ScriptQueryTimelineFiles;
std::vector<TString> InProgressStatisticsOutputFiles;

EResultOutputFormat ResultOutputFormat = EResultOutputFormat::RowsJson;
NYdb::NConsoleClient::EDataFormat PlanOutputFormat = NYdb::NConsoleClient::EDataFormat::Default;
ETraceOptType TraceOptType = ETraceOptType::Disabled;
std::optional<size_t> TraceOptScriptId;

TDuration ScriptCancelAfter;

Expand All @@ -95,6 +96,15 @@ struct TRequestOptions {
TString UserSID;
TString Database;
TDuration Timeout;
size_t QueryId = 0;
};

template <typename TValue>
static TValue GetValue(size_t index, const std::vector<TValue>& values, TValue defaultValue) {
if (values.empty()) {
return defaultValue;
}
return values[std::min(index, values.size() - 1)];
}

} // namespace NKqpRun
60 changes: 34 additions & 26 deletions ydb/tests/tools/kqprun/src/kqp_runner.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ class TKqpRunner::TImpl {
}

bool ExecuteScript(const TRequestOptions& script) {
StartScriptTraceOpt();
StartScriptTraceOpt(script.QueryId);

TRequestResult status = YdbSetup_.ScriptRequest(script, ExecutionOperation_);

Expand All @@ -132,11 +132,11 @@ class TKqpRunner::TImpl {
ExecutionMeta_ = TExecutionMeta();
ExecutionMeta_.Database = script.Database;

return WaitScriptExecutionOperation();
return WaitScriptExecutionOperation(script.QueryId);
}

bool ExecuteQuery(const TRequestOptions& query, EQueryType queryType) {
StartScriptTraceOpt();
StartScriptTraceOpt(query.QueryId);
StartTime_ = TInstant::Now();

TString queryTypeStr;
Expand Down Expand Up @@ -164,9 +164,9 @@ class TKqpRunner::TImpl {
meta.Plan = ExecutionMeta_.Plan;
}

PrintScriptAst(meta.Ast);
PrintScriptProgress(meta.Plan);
PrintScriptPlan(meta.Plan);
PrintScriptAst(query.QueryId, meta.Ast);
PrintScriptProgress(query.QueryId, meta.Plan);
PrintScriptPlan(query.QueryId, meta.Plan);
PrintScriptFinish(meta, queryTypeStr);

if (!status.IsSuccess()) {
Expand Down Expand Up @@ -233,7 +233,7 @@ class TKqpRunner::TImpl {
}

private:
bool WaitScriptExecutionOperation() {
bool WaitScriptExecutionOperation(ui64 queryId) {
StartTime_ = TInstant::Now();

TDuration getOperationPeriod = TDuration::Seconds(1);
Expand All @@ -244,7 +244,7 @@ class TKqpRunner::TImpl {
TRequestResult status;
while (true) {
status = YdbSetup_.GetScriptExecutionOperationRequest(ExecutionMeta_.Database, ExecutionOperation_, ExecutionMeta_);
PrintScriptProgress(ExecutionMeta_.Plan);
PrintScriptProgress(queryId, ExecutionMeta_.Plan);

if (ExecutionMeta_.Ready) {
break;
Expand All @@ -267,9 +267,11 @@ class TKqpRunner::TImpl {
Sleep(getOperationPeriod);
}

PrintScriptAst(ExecutionMeta_.Ast);
PrintScriptProgress(ExecutionMeta_.Plan);
PrintScriptPlan(ExecutionMeta_.Plan);
TYdbSetup::StopTraceOpt();

PrintScriptAst(queryId, ExecutionMeta_.Ast);
PrintScriptProgress(queryId, ExecutionMeta_.Plan);
PrintScriptPlan(queryId, ExecutionMeta_.Plan);
PrintScriptFinish(ExecutionMeta_, "Script");

if (!status.IsSuccess() || ExecutionMeta_.ExecutionStatus != NYdb::NQuery::EExecStatus::Completed) {
Expand All @@ -290,8 +292,14 @@ class TKqpRunner::TImpl {
}
}

void StartScriptTraceOpt() const {
if (Options_.TraceOptType == TRunnerOptions::ETraceOptType::All || Options_.TraceOptType == TRunnerOptions::ETraceOptType::Script) {
void StartScriptTraceOpt(size_t queryId) const {
bool startTraceOpt = Options_.TraceOptType == TRunnerOptions::ETraceOptType::All;

if (Options_.TraceOptType == TRunnerOptions::ETraceOptType::Script) {
startTraceOpt |= !Options_.TraceOptScriptId || *Options_.TraceOptScriptId == queryId;
}

if (startTraceOpt) {
YdbSetup_.StartTraceOpt();
}
}
Expand All @@ -303,10 +311,10 @@ class TKqpRunner::TImpl {
}
}

void PrintScriptAst(const TString& ast) const {
if (Options_.ScriptQueryAstOutput) {
void PrintScriptAst(size_t queryId, const TString& ast) const {
if (const auto output = GetValue<IOutputStream*>(queryId, Options_.ScriptQueryAstOutputs, nullptr)) {
Cout << CoutColors_.Cyan() << "Writing script query ast" << CoutColors_.Default() << Endl;
Options_.ScriptQueryAstOutput->Write(ast);
output->Write(ast);
}
}

Expand All @@ -325,16 +333,16 @@ class TKqpRunner::TImpl {
printer.Print(plan);
}

void PrintScriptPlan(const TString& plan) const {
if (Options_.ScriptQueryPlanOutput) {
void PrintScriptPlan(size_t queryId, const TString& plan) const {
if (const auto output = GetValue<IOutputStream*>(queryId, Options_.ScriptQueryPlanOutputs, nullptr)) {
Cout << CoutColors_.Cyan() << "Writing script query plan" << CoutColors_.Default() << Endl;
PrintPlan(plan, Options_.ScriptQueryPlanOutput);
PrintPlan(plan, output);
}
}

void PrintScriptProgress(const TString& plan) const {
if (Options_.InProgressStatisticsOutputFile) {
TFileOutput outputStream(Options_.InProgressStatisticsOutputFile);
void PrintScriptProgress(size_t queryId, const TString& plan) const {
if (const auto& output = GetValue<TString>(queryId, Options_.InProgressStatisticsOutputFiles, {})) {
TFileOutput outputStream(output);
outputStream << TInstant::Now().ToIsoStringLocal() << " Script in progress statistics" << Endl;

auto convertedPlan = plan;
Expand All @@ -361,8 +369,8 @@ class TKqpRunner::TImpl {

outputStream.Finish();
}
if (Options_.ScriptQueryTimelineFile) {
TFileOutput outputStream(Options_.ScriptQueryTimelineFile);
if (const auto& output = GetValue<TString>(queryId, Options_.ScriptQueryTimelineFiles, {})) {
TFileOutput outputStream(output);

TPlanVisualizer planVisualizer;
planVisualizer.LoadPlans(plan);
Expand All @@ -373,10 +381,10 @@ class TKqpRunner::TImpl {
}

TProgressCallback GetProgressCallback() {
return [this](const NKikimrKqp::TEvExecuterProgress& executerProgress) mutable {
return [this](ui64 queryId, const NKikimrKqp::TEvExecuterProgress& executerProgress) mutable {
const TString& plan = executerProgress.GetQueryPlan();
ExecutionMeta_.Plan = plan;
PrintScriptProgress(plan);
PrintScriptProgress(queryId, plan);
};
}

Expand Down
2 changes: 2 additions & 0 deletions ydb/tests/tools/kqprun/src/ya.make
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ PEERDIR(
ydb/core/testlib
)

GENERATE_ENUM_SERIALIZATION(common.h)

YQL_LAST_ABI_VERSION()

END()
3 changes: 2 additions & 1 deletion ydb/tests/tools/kqprun/src/ydb_setup.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -535,7 +535,8 @@ class TYdbSetup::TImpl {
.Event = std::move(event),
.TargetNode = GetRuntime()->GetNodeId(targetNodeIndex),
.ResultRowsLimit = Settings_.AppConfig.GetQueryServiceConfig().GetScriptResultRowsLimit(),
.ResultSizeLimit = Settings_.AppConfig.GetQueryServiceConfig().GetScriptResultSizeLimit()
.ResultSizeLimit = Settings_.AppConfig.GetQueryServiceConfig().GetScriptResultSizeLimit(),
.QueryId = query.QueryId
};
}

Expand Down

0 comments on commit 1195957

Please sign in to comment.