Add session property to set input table matching threshold

prestodb · Oct 12, 2023 · aa39d9c · aa39d9c
1 parent bb433ab
commit aa39d9c
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 4 deletions.
diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java
@@ -254,6 +254,7 @@ public final class SystemSessionProperties
     public static final String HISTORY_CANONICAL_PLAN_NODE_LIMIT = "history_canonical_plan_node_limit";
     public static final String HISTORY_BASED_OPTIMIZER_TIMEOUT_LIMIT = "history_based_optimizer_timeout_limit";
     public static final String RESTRICT_HISTORY_BASED_OPTIMIZATION_TO_COMPLEX_QUERY = "restrict_history_based_optimization_to_complex_query";
+    public static final String HISTORY_INPUT_TABLE_STATISTICS_MATCHING_THRESHOLD = "history_input_table_statistics_matching_threshold";
     public static final String MAX_LEAF_NODES_IN_PLAN = "max_leaf_nodes_in_plan";
     public static final String LEAF_NODE_LIMIT_ENABLED = "leaf_node_limit_enabled";
     public static final String PUSH_REMOTE_EXCHANGE_THROUGH_GROUP_ID = "push_remote_exchange_through_group_id";
@@ -1467,6 +1468,11 @@ public SystemSessionProperties(
                         "Enable history based optimization only for complex queries, i.e. queries with join and aggregation",
                         true,
                         false),
+                doubleProperty(
+                        HISTORY_INPUT_TABLE_STATISTICS_MATCHING_THRESHOLD,
+                        "When the size difference between current table and history table exceed this threshold, do not match history statistics",
+                        0.0,
+                        true),
                 new PropertyMetadata<>(
                         MAX_LEAF_NODES_IN_PLAN,
                         "Maximum number of leaf nodes in the logical plan of SQL statement",
@@ -2768,6 +2774,11 @@ public static boolean restrictHistoryBasedOptimizationToComplexQuery(Session ses
         return session.getSystemProperty(RESTRICT_HISTORY_BASED_OPTIMIZATION_TO_COMPLEX_QUERY, Boolean.class);
     }
 
+    public static double getHistoryInputTableStatisticsMatchingThreshold(Session session)
+    {
+        return session.getSystemProperty(HISTORY_INPUT_TABLE_STATISTICS_MATCHING_THRESHOLD, Double.class);
+    }
+
     public static boolean shouldPushRemoteExchangeThroughGroupId(Session session)
     {
         return session.getSystemProperty(PUSH_REMOTE_EXCHANGE_THROUGH_GROUP_ID, Boolean.class);

diff --git a/presto-main/src/main/java/com/facebook/presto/cost/HistoricalPlanStatisticsUtil.java b/presto-main/src/main/java/com/facebook/presto/cost/HistoricalPlanStatisticsUtil.java
@@ -34,14 +34,14 @@ private HistoricalPlanStatisticsUtil() {}
     public static PlanStatistics getPredictedPlanStatistics(
             HistoricalPlanStatistics historicalPlanStatistics,
             List<PlanStatistics> inputTableStatistics,
-            HistoryBasedOptimizationConfig config)
+            double historyMatchingThreshold)
     {
         List<HistoricalPlanStatisticsEntry> lastRunsStatistics = historicalPlanStatistics.getLastRunsStatistics();
         if (lastRunsStatistics.isEmpty()) {
             return PlanStatistics.empty();
         }
 
-        Optional<Integer> similarStatsIndex = getSimilarStatsIndex(historicalPlanStatistics, inputTableStatistics, config.getHistoryMatchingThreshold());
+        Optional<Integer> similarStatsIndex = getSimilarStatsIndex(historicalPlanStatistics, inputTableStatistics, historyMatchingThreshold);
 
         if (similarStatsIndex.isPresent()) {
             return lastRunsStatistics.get(similarStatsIndex.get()).getPlanStatistics();

diff --git a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java
@@ -35,6 +35,7 @@
 import java.util.function.Supplier;
 
 import static com.facebook.presto.SystemSessionProperties.getHistoryBasedOptimizerTimeoutLimit;
+import static com.facebook.presto.SystemSessionProperties.getHistoryInputTableStatisticsMatchingThreshold;
 import static com.facebook.presto.SystemSessionProperties.useHistoryBasedPlanStatisticsEnabled;
 import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.historyBasedPlanCanonicalizationStrategyList;
 import static com.facebook.presto.cost.HistoricalPlanStatisticsUtil.getPredictedPlanStatistics;
@@ -157,13 +158,14 @@ private PlanNodeStatsEstimate getStatistics(PlanNode planNode, Session session,
         catch (ExecutionException e) {
             throw new RuntimeException(format("Unable to get plan statistics for %s", planNode), e.getCause());
         }
+        double historyMatchingThreshold = getHistoryInputTableStatisticsMatchingThreshold(session) > 0 ? getHistoryInputTableStatisticsMatchingThreshold(session) : config.getHistoryMatchingThreshold();
         // Return statistics corresponding to first strategy that we find, in order specified by `historyBasedPlanCanonicalizationStrategyList`
         for (PlanCanonicalizationStrategy strategy : historyBasedPlanCanonicalizationStrategyList()) {
             for (Map.Entry<PlanNodeWithHash, HistoricalPlanStatistics> entry : statistics.entrySet()) {
                 if (allHashes.containsKey(strategy) && entry.getKey().getHash().isPresent() && allHashes.get(strategy).equals(entry.getKey())) {
                     Optional<List<PlanStatistics>> inputTableStatistics = getPlanNodeInputTableStatistics(plan, session);
                     if (inputTableStatistics.isPresent()) {
-                        PlanStatistics predictedPlanStatistics = getPredictedPlanStatistics(entry.getValue(), inputTableStatistics.get(), config);
+                        PlanStatistics predictedPlanStatistics = getPredictedPlanStatistics(entry.getValue(), inputTableStatistics.get(), historyMatchingThreshold);
                         if (predictedPlanStatistics.getConfidence() > 0) {
                             return delegateStats.combineStats(
                                     predictedPlanStatistics,

diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestHistoricalPlanStatistics.java b/presto-main/src/test/java/com/facebook/presto/cost/TestHistoricalPlanStatistics.java
@@ -106,6 +106,6 @@ private static PlanStatistics getPredictedPlanStatistics(
         return HistoricalPlanStatisticsUtil.getPredictedPlanStatistics(
                 historicalPlanStatistics,
                 inputTableStatistics,
-                new HistoryBasedOptimizationConfig());
+                0.1);
     }
 }