From b358641527e94a3a8a42a3f1d23011a466095bcb Mon Sep 17 00:00:00 2001 From: Bikramjeet Vig Date: Wed, 4 Oct 2023 12:02:04 -0700 Subject: [PATCH] [Native] Add debug config for native engine execution This patch adds the required changes to allow setting a debug query config specific to native engine. The query config added on presto side is `native_debug.validate_output_from_operators` Test Plan: Updated and added unit tests --- .../presto/SystemSessionProperties.java | 10 ++++++++++ .../main/tests/QueryContextManagerTest.cpp | 19 +++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index 440bfa20c0a46..50c70eae2d84a 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -306,6 +306,7 @@ public final class SystemSessionProperties public static final String NATIVE_EXECUTION_EXECUTABLE_PATH = "native_execution_executable_path"; public static final String NATIVE_EXECUTION_PROGRAM_ARGUMENTS = "native_execution_program_arguments"; public static final String NATIVE_EXECUTION_PROCESS_REUSE_ENABLED = "native_execution_process_reuse_enabled"; + public static final String NATIVE_DEBUG_VALIDATE_OUTPUT_FROM_OPERATORS = "native_debug.validate_output_from_operators"; private final List> sessionProperties; @@ -1561,6 +1562,15 @@ public SystemSessionProperties( "Enable reuse the native process within the same JVM", true, false), + booleanProperty( + NATIVE_DEBUG_VALIDATE_OUTPUT_FROM_OPERATORS, + "If set to true, then during execution of tasks, the output vectors of " + + "every operator are validated for consistency. This is an expensive check " + + "so should only be used for debugging. It can help debug issues where " + + "malformed vector cause failures or crashes by helping identify which " + + "operator is generating them.", + false, + true), booleanProperty( RANDOMIZE_OUTER_JOIN_NULL_KEY, "(Deprecated) Randomize null join key for outer join", diff --git a/presto-native-execution/presto_cpp/main/tests/QueryContextManagerTest.cpp b/presto-native-execution/presto_cpp/main/tests/QueryContextManagerTest.cpp index 570c25444affb..c0771ec873eff 100644 --- a/presto-native-execution/presto_cpp/main/tests/QueryContextManagerTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/QueryContextManagerTest.cpp @@ -31,16 +31,31 @@ TEST_F(QueryContextManagerTest, nativeSessionProperties) { .systemProperties = { {"native_max_spill_level", "1"}, {"native_spill_compression_codec", "NONE"}, - {"native_join_spill_enabled", "true"}, + {"native_join_spill_enabled", "false"}, {"native_spill_write_buffer_size", "1024"}, + {"native_debug.validate_output_from_operators", "true"}, {"aggregation_spill_all", "true"}}}; auto queryCtx = taskManager_->getQueryContextManager()->findOrCreateQueryCtx( taskId, session); EXPECT_EQ(queryCtx->queryConfig().maxSpillLevel(), 1); EXPECT_EQ(queryCtx->queryConfig().spillCompressionKind(), "NONE"); - EXPECT_TRUE(queryCtx->queryConfig().joinSpillEnabled()); + EXPECT_FALSE(queryCtx->queryConfig().joinSpillEnabled()); + EXPECT_TRUE(queryCtx->queryConfig().validateOutputFromOperators()); EXPECT_EQ(queryCtx->queryConfig().spillWriteBufferSize(), 1024); EXPECT_TRUE(queryCtx->queryConfig().aggregationSpillAll()); } +TEST_F(QueryContextManagerTest, defaultSessionProperties) { + protocol::TaskId taskId = "scan.0.0.1.0"; + protocol::SessionRepresentation session{.systemProperties = {}}; + auto queryCtx = taskManager_->getQueryContextManager()->findOrCreateQueryCtx( + taskId, session); + EXPECT_EQ(queryCtx->queryConfig().maxSpillLevel(), 4); + EXPECT_EQ(queryCtx->queryConfig().spillCompressionKind(), "none"); + EXPECT_TRUE(queryCtx->queryConfig().joinSpillEnabled()); + EXPECT_FALSE(queryCtx->queryConfig().validateOutputFromOperators()); + EXPECT_EQ(queryCtx->queryConfig().spillWriteBufferSize(), 1L << 20); + EXPECT_FALSE(queryCtx->queryConfig().aggregationSpillAll()); +} + } // namespace facebook::presto