From 32b6c9f84829963d3b407a2d8b489929a1ecda84 Mon Sep 17 00:00:00 2001
From: Pang Wu <104795337+pang-wu@users.noreply.github.com>
Date: Thu, 21 Dec 2023 19:03:11 -0800
Subject: [PATCH] Support Spark 3.4.2, lift pyspark dependency range (#397)

---
 .../main/scala/com/intel/raydp/shims/SparkShimProvider.scala   | 3 ++-
 python/setup.py                                                | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala b/core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala
index a73dcacf..28e0c7ed 100644
--- a/core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala
+++ b/core/shims/spark340/src/main/scala/com/intel/raydp/shims/SparkShimProvider.scala
@@ -22,7 +22,8 @@ import com.intel.raydp.shims.{SparkShims, SparkShimDescriptor}
 object SparkShimProvider {
   val SPARK340_DESCRIPTOR = SparkShimDescriptor(3, 4, 0)
   val SPARK341_DESCRIPTOR = SparkShimDescriptor(3, 4, 1)
-  val DESCRIPTOR_STRINGS = Seq(s"$SPARK340_DESCRIPTOR", s"$SPARK341_DESCRIPTOR")
+  val SPARK342_DESCRIPTOR = SparkShimDescriptor(3, 4, 2)
+  val DESCRIPTOR_STRINGS = Seq(s"$SPARK340_DESCRIPTOR", s"$SPARK341_DESCRIPTOR", s"$SPARK342_DESCRIPTOR")
   val DESCRIPTOR = SPARK341_DESCRIPTOR
 }
 
diff --git a/python/setup.py b/python/setup.py
index 4af2ed44..b5231785 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -100,7 +100,7 @@ def run(self):
         "psutil",
         "pyarrow >= 4.0.1",
         "ray >= 2.1.0",
-        "pyspark >= 3.1.1, <= 3.4.1",
+        "pyspark >= 3.1.1, <= 3.5.0",
         "netifaces",
         "protobuf > 3.19.5, <= 3.20.3"
     ]