From f71f355eb686ae88523d962622fea2b48fcc2559 Mon Sep 17 00:00:00 2001 From: NoeB Date: Thu, 14 Nov 2024 00:57:42 +0100 Subject: [PATCH] feat: support array_append (#1072) * feat: support array_append * formatted code * rewrite array_append plan to match spark behaviour and fixed bug in QueryPlan serde * remove unwrap * Fix for Spark 3.3 * refactor array_append binary expression serde code * Disabled array_append test for spark 4.0+ --- .../src/test/scala/org/apache/comet/CometExpressionSuite.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 5653eee01..c9ca29327 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -2544,7 +2544,8 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("array_append") { - assume(isSpark34Plus) + // array append has been added in Spark 3.4 and in Spark 4.0 it gets written to ArrayInsert + assume(isSpark34Plus && !isSpark40Plus) Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet")