Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SPARK-21603][SQL][FOLLOW-UP] Use -1 to disable maxLinesPerFunction #19031

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -357,15 +357,19 @@ class CodegenContext {

/**
* It will count the lines of every Java function generated by whole-stage codegen,
* if there is a function of length greater than spark.sql.codegen.maxLinesPerFunction,
* it will return true.
* if there is a function of length greater than `maxLinesPerFunction`, it will return true.
* If `maxLinesPerFunction` has -1, it will always return false.
*/
def isTooLongGeneratedFunction: Boolean = {
classFunctions.values.exists { _.values.exists {
code =>
val codeWithoutComments = CodeFormatter.stripExtraNewLinesAndComments(code)
codeWithoutComments.count(_ == '\n') > SQLConf.get.maxLinesPerFunction
val maxLinesPerFunc = SQLConf.get.maxLinesPerFunction
if (maxLinesPerFunc >= 0) {
classFunctions.values.exists { _.values.exists { code =>
val codeWithoutComments = CodeFormatter.stripExtraNewLinesAndComments(code)
codeWithoutComments.count(_ == '\n') > maxLinesPerFunc
}
}
} else {
false
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -578,8 +578,10 @@ object SQLConf {
"When the generated function exceeds this threshold, " +
"the whole-stage codegen is deactivated for this subtree of the current query plan. " +
"The default value 4000 is the max length of byte code JIT supported " +
"for a single function(8000) divided by 2.")
"for a single function(8000) divided by 2. Use -1 to disable this.")
.intConf
.checkValue(maxLines => maxLines >= -1, "The maximum must not be a negative integer, " +
"except for -1 using to always activate whole-stage codegen")
.createWithDefault(4000)

val FILES_MAX_PARTITION_BYTES = buildConf("spark.sql.files.maxPartitionBytes")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@

package org.apache.spark.sql.execution

import org.apache.spark.sql.{Column, Dataset, Row}
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
import org.apache.spark.sql.catalyst.expressions.{Add, Literal, Stack}
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
import org.apache.spark.sql.execution.aggregate.HashAggregateExec
import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
Expand Down Expand Up @@ -193,11 +191,15 @@ class WholeStageCodegenSuite extends SparkPlanTest with SharedSQLContext {
}
}

test("SPARK-21603 check there is not a too long generated function when threshold is Int.Max") {
test("SPARK-21603 check there is not a too long generated function when threshold is Max/-1") {
withSQLConf(SQLConf.WHOLESTAGE_MAX_LINES_PER_FUNCTION.key -> Int.MaxValue.toString) {
val ctx = genGroupByCodeGenContext(30)
assert(ctx.isTooLongGeneratedFunction === false)
}
withSQLConf(SQLConf.WHOLESTAGE_MAX_LINES_PER_FUNCTION.key -> "-1") {
val ctx = genGroupByCodeGenContext(30)
assert(ctx.isTooLongGeneratedFunction === false)
}
}

test("SPARK-21603 check there is a too long generated function when threshold is 0") {
Expand All @@ -206,4 +208,12 @@ class WholeStageCodegenSuite extends SparkPlanTest with SharedSQLContext {
assert(ctx.isTooLongGeneratedFunction === true)
}
}

test("SPARK-21603 `maxLinesPerFunction` must not be negative") {
val errMsg = intercept[IllegalArgumentException] {
withSQLConf(SQLConf.WHOLESTAGE_MAX_LINES_PER_FUNCTION.key -> "-2") {}
}.getMessage
assert(errMsg.contains("The maximum must not be a negative integer, except for -1 using to " +
"always activate whole-stage codegen"))
}
}