[FEATURE-apache#10] Uses sort columnar shuffle for native columnar sh…

…uffle by default (merge request !16)
sharkdtu · Nov 8, 2024 · fd1a807 · fd1a807
1 parent 21291f3
commit fd1a807
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
@@ -169,6 +169,10 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging {
     if (!conf.contains(GlutenConfig.NATIVE_WRITER_ENABLED.key)) {
       conf.set(GlutenConfig.NATIVE_WRITER_ENABLED.key, "true")
     }
+
+    if (!conf.contains(GlutenConfig.COLUMNAR_SHUFFLE_SORT_COLUMNS_THRESHOLD.key)) {
+      conf.set(GlutenConfig.COLUMNAR_SHUFFLE_SORT_COLUMNS_THRESHOLD.key, "200")
+    }
   }
 
   private def setPredefinedConfigs(sc: SparkContext, conf: SparkConf): Unit = {