diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index 88d9d6dc76b6c..3c71888e03e07 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -254,6 +254,7 @@ public final class SystemSessionProperties public static final String HISTORY_CANONICAL_PLAN_NODE_LIMIT = "history_canonical_plan_node_limit"; public static final String HISTORY_BASED_OPTIMIZER_TIMEOUT_LIMIT = "history_based_optimizer_timeout_limit"; public static final String RESTRICT_HISTORY_BASED_OPTIMIZATION_TO_COMPLEX_QUERY = "restrict_history_based_optimization_to_complex_query"; + public static final String HISTORY_INPUT_TABLE_STATISTICS_MATCHING_THRESHOLD = "history_input_table_statistics_matching_threshold"; public static final String MAX_LEAF_NODES_IN_PLAN = "max_leaf_nodes_in_plan"; public static final String LEAF_NODE_LIMIT_ENABLED = "leaf_node_limit_enabled"; public static final String PUSH_REMOTE_EXCHANGE_THROUGH_GROUP_ID = "push_remote_exchange_through_group_id"; @@ -1467,6 +1468,11 @@ public SystemSessionProperties( "Enable history based optimization only for complex queries, i.e. queries with join and aggregation", true, false), + doubleProperty( + HISTORY_INPUT_TABLE_STATISTICS_MATCHING_THRESHOLD, + "When the size difference between current table and history table exceed this threshold, do not match history statistics", + 0.0, + true), new PropertyMetadata<>( MAX_LEAF_NODES_IN_PLAN, "Maximum number of leaf nodes in the logical plan of SQL statement", @@ -2768,6 +2774,11 @@ public static boolean restrictHistoryBasedOptimizationToComplexQuery(Session ses return session.getSystemProperty(RESTRICT_HISTORY_BASED_OPTIMIZATION_TO_COMPLEX_QUERY, Boolean.class); } + public static double getHistoryInputTableStatisticsMatchingThreshold(Session session) + { + return session.getSystemProperty(HISTORY_INPUT_TABLE_STATISTICS_MATCHING_THRESHOLD, Double.class); + } + public static boolean shouldPushRemoteExchangeThroughGroupId(Session session) { return session.getSystemProperty(PUSH_REMOTE_EXCHANGE_THROUGH_GROUP_ID, Boolean.class); diff --git a/presto-main/src/main/java/com/facebook/presto/cost/HistoricalPlanStatisticsUtil.java b/presto-main/src/main/java/com/facebook/presto/cost/HistoricalPlanStatisticsUtil.java index c9cab9afac3cf..dbad0928beaa6 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/HistoricalPlanStatisticsUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/HistoricalPlanStatisticsUtil.java @@ -34,14 +34,14 @@ private HistoricalPlanStatisticsUtil() {} public static PlanStatistics getPredictedPlanStatistics( HistoricalPlanStatistics historicalPlanStatistics, List inputTableStatistics, - HistoryBasedOptimizationConfig config) + double historyMatchingThreshold) { List lastRunsStatistics = historicalPlanStatistics.getLastRunsStatistics(); if (lastRunsStatistics.isEmpty()) { return PlanStatistics.empty(); } - Optional similarStatsIndex = getSimilarStatsIndex(historicalPlanStatistics, inputTableStatistics, config.getHistoryMatchingThreshold()); + Optional similarStatsIndex = getSimilarStatsIndex(historicalPlanStatistics, inputTableStatistics, historyMatchingThreshold); if (similarStatsIndex.isPresent()) { return lastRunsStatistics.get(similarStatsIndex.get()).getPlanStatistics(); diff --git a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java index 7a13bd079f5bb..6e186d570a1f4 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java @@ -35,6 +35,7 @@ import java.util.function.Supplier; import static com.facebook.presto.SystemSessionProperties.getHistoryBasedOptimizerTimeoutLimit; +import static com.facebook.presto.SystemSessionProperties.getHistoryInputTableStatisticsMatchingThreshold; import static com.facebook.presto.SystemSessionProperties.useHistoryBasedPlanStatisticsEnabled; import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.historyBasedPlanCanonicalizationStrategyList; import static com.facebook.presto.cost.HistoricalPlanStatisticsUtil.getPredictedPlanStatistics; @@ -157,13 +158,14 @@ private PlanNodeStatsEstimate getStatistics(PlanNode planNode, Session session, catch (ExecutionException e) { throw new RuntimeException(format("Unable to get plan statistics for %s", planNode), e.getCause()); } + double historyMatchingThreshold = getHistoryInputTableStatisticsMatchingThreshold(session) > 0 ? getHistoryInputTableStatisticsMatchingThreshold(session) : config.getHistoryMatchingThreshold(); // Return statistics corresponding to first strategy that we find, in order specified by `historyBasedPlanCanonicalizationStrategyList` for (PlanCanonicalizationStrategy strategy : historyBasedPlanCanonicalizationStrategyList()) { for (Map.Entry entry : statistics.entrySet()) { if (allHashes.containsKey(strategy) && entry.getKey().getHash().isPresent() && allHashes.get(strategy).equals(entry.getKey())) { Optional> inputTableStatistics = getPlanNodeInputTableStatistics(plan, session); if (inputTableStatistics.isPresent()) { - PlanStatistics predictedPlanStatistics = getPredictedPlanStatistics(entry.getValue(), inputTableStatistics.get(), config); + PlanStatistics predictedPlanStatistics = getPredictedPlanStatistics(entry.getValue(), inputTableStatistics.get(), historyMatchingThreshold); if (predictedPlanStatistics.getConfidence() > 0) { return delegateStats.combineStats( predictedPlanStatistics, diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestHistoricalPlanStatistics.java b/presto-main/src/test/java/com/facebook/presto/cost/TestHistoricalPlanStatistics.java index 870a3012422d5..05cee3a47d32b 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestHistoricalPlanStatistics.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestHistoricalPlanStatistics.java @@ -106,6 +106,6 @@ private static PlanStatistics getPredictedPlanStatistics( return HistoricalPlanStatisticsUtil.getPredictedPlanStatistics( historicalPlanStatistics, inputTableStatistics, - new HistoryBasedOptimizationConfig()); + 0.1); } }