From 72f73fb8ebad3f48cf77515abdad4be866efafbb Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Wed, 23 Dec 2015 14:56:10 -0800 Subject: [PATCH] combine the limits. --- .../spark/sql/catalyst/optimizer/Optimizer.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index d3bbac85082e8..48286d7b42ffa 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -868,11 +868,15 @@ object PushLimitThroughOuterJoin extends Rule[LogicalPlan] with PredicateHelper case f @ Limit(expr, Join(left, right, joinType, joinCondition)) => joinType match { case RightOuter => - Limit(expr, Join(left, Limit(expr, right), joinType, joinCondition)) + Limit(expr, Join(left, CombineLimits(Limit(expr, right)), joinType, joinCondition)) case LeftOuter => - Limit(expr, Join(Limit(expr, left), right, joinType, joinCondition)) + Limit(expr, Join(CombineLimits(Limit(expr, left)), right, joinType, joinCondition)) case FullOuter => - Limit(expr, Join(Limit(expr, left), Limit(expr, right), joinType, joinCondition)) + Limit(expr, + Join( + CombineLimits(Limit(expr, left)), + CombineLimits(Limit(expr, right)), + joinType, joinCondition)) case _ => f // DO Nothing for the other join types } }