[ML-311] Upgrade spark to 3.2.2 to resolve security scan

Signed-off-by: minmingzhu <[email protected]>
oap-project · Jun 30, 2023 · 392d0c1 · 392d0c1
1 parent a2122a8
commit 392d0c1
Show file tree

Hide file tree

Showing 28 changed files with 38 additions and 36 deletions.
diff --git a/README.md b/README.md
@@ -73,6 +73,7 @@ The following runtime packages with all their dependencies should be installed i
 * Apache Spark 3.1.3
 * Apache Spark 3.2.0
 * Apache Spark 3.2.1
+* Apache Spark 3.2.2
 
 ### Supported Intel® oneAPI Toolkits
 

diff --git a/RELEASE b/RELEASE
@@ -1,3 +1,3 @@
 OAP_MLLIB_VERSION=1.6.0
-SPARK_VERSION=3.2.0
+SPARK_VERSION=3.2.2
 PLATFORM_PROFILE=CPU_GPU_PROFILE
diff --git a/examples/scala/als-scala/pom.xml b/examples/scala/als-scala/pom.xml
@@ -15,7 +15,7 @@
     <oap.version>1.2.0</oap.version>
     <scala.version>2.12.15</scala.version>
     <scala.binary.version>2.12</scala.binary.version>
-    <spark.version>3.2.0</spark.version>
+    <spark.version>3.2.2</spark.version>
   </properties>
 
   <dependencies>

diff --git a/examples/scala/correlation-scala/pom.xml b/examples/scala/correlation-scala/pom.xml
@@ -15,7 +15,7 @@
     <oap.version>1.2.0</oap.version>
     <scala.version>2.12.15</scala.version>
     <scala.binary.version>2.12</scala.binary.version>
-    <spark.version>3.2.0</spark.version>
+    <spark.version>3.2.2</spark.version>
   </properties>
 
   <dependencies>

diff --git a/examples/scala/kmeans-scala/pom.xml b/examples/scala/kmeans-scala/pom.xml
@@ -15,7 +15,7 @@
     <oap.version>1.2.0</oap.version>
     <scala.version>2.12.15</scala.version>
     <scala.binary.version>2.12</scala.binary.version>
-    <spark.version>3.2.0</spark.version>
+    <spark.version>3.2.2</spark.version>
   </properties>
 
   <dependencies>

diff --git a/examples/scala/linear-regression-scala/pom.xml b/examples/scala/linear-regression-scala/pom.xml
@@ -15,7 +15,7 @@
     <oap.version>1.2.0</oap.version>
     <scala.version>2.12.15</scala.version>
     <scala.binary.version>2.12</scala.binary.version>
-    <spark.version>3.2.0</spark.version>
+    <spark.version>3.2.2</spark.version>
   </properties>
 
   <dependencies>

diff --git a/examples/scala/naive-bayes-scala/pom.xml b/examples/scala/naive-bayes-scala/pom.xml
@@ -15,7 +15,7 @@
     <oap.version>1.2.0</oap.version>
     <scala.version>2.12.15</scala.version>
     <scala.binary.version>2.12</scala.binary.version>
-    <spark.version>3.2.0</spark.version>
+    <spark.version>3.2.2</spark.version>
   </properties>
 
   <dependencies>

diff --git a/examples/scala/pca-scala/pom.xml b/examples/scala/pca-scala/pom.xml
@@ -15,7 +15,7 @@
     <oap.version>1.2.0</oap.version>
     <scala.version>2.12.15</scala.version>
     <scala.binary.version>2.12</scala.binary.version>
-    <spark.version>3.2.0</spark.version>
+    <spark.version>3.2.2</spark.version>
   </properties>
 
   <dependencies>

diff --git a/examples/scala/summarizer-scala/pom.xml b/examples/scala/summarizer-scala/pom.xml
@@ -15,7 +15,7 @@
     <oap.version>1.3.0</oap.version>
     <scala.version>2.12.15</scala.version>
     <scala.binary.version>2.12</scala.binary.version>
-    <spark.version>3.2.0</spark.version>
+    <spark.version>3.2.2</spark.version>
   </properties>
 
   <dependencies>

diff --git a/mllib-dal/pom.xml b/mllib-dal/pom.xml
@@ -16,7 +16,7 @@
         <scala.version>2.12.15</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
         <scalatest.version>3.2.9</scalatest.version>
-        <spark.version>3.2.0</spark.version>
+        <spark.version>3.2.2</spark.version>
         <dal.version>2021.5.0.91</dal.version>
         <assembly.description>src/assembly/assembly.xml</assembly.description>
     </properties>

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/classification/NaiveBayesShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/classification/NaiveBayesShim.scala
@@ -20,7 +20,7 @@ import com.intel.oap.mllib.Utils
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.classification.NaiveBayesModel
-import org.apache.spark.ml.classification.spark321.{NaiveBayes => NaiveBayesSpark321}
+import org.apache.spark.ml.classification.spark322.{NaiveBayes => NaiveBayesSpark322}
 import org.apache.spark.ml.param.ParamMap
 import org.apache.spark.sql.Dataset
 import org.apache.spark.{SPARK_VERSION, SparkException}
@@ -35,7 +35,7 @@ object NaiveBayesShim extends Logging {
     logInfo(s"Loading NaiveBayes for Spark $SPARK_VERSION")
 
     val shim = Utils.getSparkVersion() match {
-      case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" => new NaiveBayesSpark321(uid)
+      case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" => new NaiveBayesSpark322(uid)
       case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION")
     }
     shim

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/classification/RandomForestClassifierShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/classification/RandomForestClassifierShim.scala
@@ -19,7 +19,7 @@ import com.intel.oap.mllib.Utils
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.classification.RandomForestClassificationModel
 import org.apache.spark.{SPARK_VERSION, SparkException}
-import org.apache.spark.ml.classification.spark321.{RandomForestClassifier => RandomForestClassifier321}
+import org.apache.spark.ml.classification.spark322.{RandomForestClassifier => RandomForestClassifier322}
 import org.apache.spark.ml.param.ParamMap
 import org.apache.spark.sql.Dataset
 
@@ -33,7 +33,8 @@ object RandomForestClassifierShim extends Logging {
     logInfo(s"Loading RandomForestClassifier for Spark $SPARK_VERSION")
 
     val shim = Utils.getSparkVersion() match {
-      case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" => new RandomForestClassifier321(uid)
+      case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" =>
+        new RandomForestClassifier322(uid)
       case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION")
     }
     shim

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/clustering/KMeansShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/clustering/KMeansShim.scala
@@ -20,7 +20,7 @@ import com.intel.oap.mllib.Utils
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.clustering.{KMeans, KMeansModel}
-import org.apache.spark.ml.clustering.spark321.{KMeans => KMeansSpark321}
+import org.apache.spark.ml.clustering.spark322.{KMeans => KMeansSpark322}
 import org.apache.spark.ml.param.ParamMap
 import org.apache.spark.sql.Dataset
 import org.apache.spark.{SPARK_VERSION, SparkException}
@@ -34,7 +34,7 @@ object KMeansShim extends Logging {
   def create(uid: String): KMeansShim = {
     logInfo(s"Loading KMeans for Spark $SPARK_VERSION")
     val kmeans = Utils.getSparkVersion() match {
-      case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" => new KMeansSpark321(uid)
+      case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" => new KMeansSpark322(uid)
       case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION")
     }
     kmeans

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/feature/PCAShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/feature/PCAShim.scala
@@ -20,7 +20,7 @@ import com.intel.oap.mllib.Utils
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.feature.PCAModel
-import org.apache.spark.ml.feature.spark321.{PCA => PCASpark321}
+import org.apache.spark.ml.feature.spark322.{PCA => PCASpark322}
 import org.apache.spark.ml.param.ParamMap
 import org.apache.spark.sql.Dataset
 import org.apache.spark.{SPARK_VERSION, SparkException}
@@ -34,7 +34,7 @@ object PCAShim extends Logging {
   def create(uid: String): PCAShim = {
     logInfo(s"Loading PCA for Spark $SPARK_VERSION")
     val pca = Utils.getSparkVersion() match {
-      case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" => new PCASpark321(uid)
+      case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" => new PCASpark322(uid)
       case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION")
     }
     pca

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/recommendation/ALSShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/recommendation/ALSShim.scala
@@ -21,7 +21,7 @@ import com.intel.oap.mllib.Utils
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.recommendation.ALS.Rating
 import org.apache.spark.ml.recommendation.spark313.{ALS => ALSSpark313}
-import org.apache.spark.ml.recommendation.spark321.{ALS => ALSSpark321}
+import org.apache.spark.ml.recommendation.spark322.{ALS => ALSSpark322}
 import org.apache.spark.rdd.RDD
 import org.apache.spark.storage.StorageLevel
 import org.apache.spark.{SPARK_VERSION, SparkException}
@@ -50,7 +50,7 @@ object ALSShim extends Logging {
     logInfo(s"Loading ALS for Spark $SPARK_VERSION")
     val als = Utils.getSparkVersion() match {
       case "3.1.1" | "3.1.2" | "3.1.3" => new ALSSpark313()
-      case "3.2.0" | "3.2.1" => new ALSSpark321()
+      case "3.2.0" | "3.2.1" | "3.2.2" => new ALSSpark322()
       case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION")
     }
     als

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/LinearRegressionShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/LinearRegressionShim.scala
@@ -22,7 +22,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.ml.param.ParamMap
 import org.apache.spark.ml.regression.LinearRegressionModel
 import org.apache.spark.ml.regression.spark313.{LinearRegression => LinearRegressionSpark313}
-import org.apache.spark.ml.regression.spark321.{LinearRegression => LinearRegressionSpark321}
+import org.apache.spark.ml.regression.spark322.{LinearRegression => LinearRegressionSpark322}
 import org.apache.spark.sql.Dataset
 import org.apache.spark.{SPARK_VERSION, SparkException}
 
@@ -36,7 +36,7 @@ object LinearRegressionShim extends Logging {
     logInfo(s"Loading ALS for Spark $SPARK_VERSION")
     val linearRegression = Utils.getSparkVersion() match {
       case "3.1.1" | "3.1.2" | "3.1.3" => new LinearRegressionSpark313(uid)
-      case "3.2.0" | "3.2.1" => new LinearRegressionSpark321(uid)
+      case "3.2.0" | "3.2.1" | "3.2.2" => new LinearRegressionSpark322(uid)
       case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION")
     }
     linearRegression

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/RandomForestRegressorShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/regression/RandomForestRegressorShim.scala
@@ -20,7 +20,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.{SPARK_VERSION, SparkException}
 import org.apache.spark.ml.param.ParamMap
 import org.apache.spark.ml.regression.RandomForestRegressionModel
-import org.apache.spark.ml.regression.spark321.{RandomForestRegressor => RandomForestRegressor321}
+import org.apache.spark.ml.regression.spark322.{RandomForestRegressor => RandomForestRegressor322}
 import org.apache.spark.sql.Dataset
 
 trait RandomForestRegressorShim extends Logging {
@@ -33,7 +33,7 @@ object RandomForestRegressorShim extends Logging {
     logInfo(s"Loading RandomForestClassifier for Spark $SPARK_VERSION")
 
     val shim = Utils.getSparkVersion() match {
-      case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" => new RandomForestRegressor321(uid)
+      case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" => new RandomForestRegressor322(uid)
       case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION")
     }
     shim

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/CorrelationShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/CorrelationShim.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.{DataFrame, Dataset}
 import org.apache.spark.storage.StorageLevel
 
 import scala.reflect.ClassTag
-import org.apache.spark.ml.stat.spark321.{Correlation => CorrelationSpark321}
+import org.apache.spark.ml.stat.spark322.{Correlation => CorrelationSpark322}
 
 trait CorrelationShim extends Serializable with Logging {
   def corr(dataset: Dataset[_], column: String, method: String): DataFrame
@@ -35,7 +35,7 @@ object CorrelationShim extends Logging {
   def create(): CorrelationShim = {
     logInfo(s"Loading Correlation for Spark $SPARK_VERSION")
     val als = Utils.getSparkVersion() match {
-      case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" => new CorrelationSpark321()
+      case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" => new CorrelationSpark322()
       case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION")
     }
     als

diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/SummarizerShim.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/stat/SummarizerShim.scala
@@ -24,7 +24,7 @@ import org.apache.spark.mllib.linalg.Vector
 import org.apache.spark.mllib.stat.MultivariateStatisticalSummary
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Dataset}
-import org.apache.spark.mllib.stat.spark321.{Statistics => SummarizerSpark321}
+import org.apache.spark.mllib.stat.spark322.{Statistics => SummarizerSpark322}
 
 trait SummarizerShim extends Serializable with Logging {
   def colStats(X: RDD[Vector]): MultivariateStatisticalSummary
@@ -35,7 +35,7 @@ object SummarizerShim extends Logging {
   def create(): SummarizerShim = {
     logInfo(s"Loading Summarizer for Spark $SPARK_VERSION")
     val summarizer = Utils.getSparkVersion() match {
-      case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" => new SummarizerSpark321()
+      case "3.1.1" | "3.1.2" | "3.1.3" | "3.2.0" | "3.2.1" | "3.2.2" => new SummarizerSpark322()
       case _ => throw new SparkException(s"Unsupported Spark version $SPARK_VERSION")
     }
     summarizer

diff --git a/.../classification/spark321/NaiveBayes.scala → .../classification/spark322/NaiveBayes.scala b/.../classification/spark321/NaiveBayes.scala → .../classification/spark322/NaiveBayes.scala
@@ -17,7 +17,7 @@
  */
 // scalastyle:on
 
-package org.apache.spark.ml.classification.spark321
+package org.apache.spark.ml.classification.spark322
 
 import com.intel.oap.mllib.Utils
 import com.intel.oap.mllib.classification.{NaiveBayesDALImpl, NaiveBayesShim}

diff --git a/...ion/spark321/RandomForestClassifier.scala → ...ion/spark322/RandomForestClassifier.scala b/...ion/spark321/RandomForestClassifier.scala → ...ion/spark322/RandomForestClassifier.scala
@@ -17,7 +17,7 @@
  */
 // scalastyle:on
 
-package org.apache.spark.ml.classification.spark321
+package org.apache.spark.ml.classification.spark322
 
 import com.intel.oap.mllib.Utils
 import com.intel.oap.mllib.classification.{LearningNode => LearningNodeDAL, RandomForestClassifierDALImpl, RandomForestClassifierShim}

diff --git a/...spark/ml/clustering/spark321/KMeans.scala → ...spark/ml/clustering/spark322/KMeans.scala b/...spark/ml/clustering/spark321/KMeans.scala → ...spark/ml/clustering/spark322/KMeans.scala
@@ -17,7 +17,7 @@
  */
 // scalastyle:on
 
-package org.apache.spark.ml.clustering.spark321
+package org.apache.spark.ml.clustering.spark322
 
 import com.intel.oap.mllib.Utils
 import com.intel.oap.mllib.clustering.{KMeansDALImpl, KMeansShim}

diff --git a/...pache/spark/ml/feature/spark321/PCA.scala → ...pache/spark/ml/feature/spark322/PCA.scala b/...pache/spark/ml/feature/spark321/PCA.scala → ...pache/spark/ml/feature/spark322/PCA.scala
@@ -17,7 +17,7 @@
  */
 // scalastyle:on
 
-package org.apache.spark.ml.feature.spark321
+package org.apache.spark.ml.feature.spark322
 
 import com.intel.oap.mllib.Utils
 import com.intel.oap.mllib.feature.{PCADALImpl, PCAShim}

diff --git a/...park/ml/recommendation/spark321/ALS.scala → ...park/ml/recommendation/spark322/ALS.scala b/...park/ml/recommendation/spark321/ALS.scala → ...park/ml/recommendation/spark322/ALS.scala
@@ -17,7 +17,7 @@
  */
 // scalastyle:on
 
-package org.apache.spark.ml.recommendation.spark321
+package org.apache.spark.ml.recommendation.spark322
 
 import com.github.fommil.netlib.BLAS.{getInstance => blas}
 import com.intel.oap.mllib.{Utils => DALUtils}

diff --git a/...egression/spark321/LinearRegression.scala → ...egression/spark322/LinearRegression.scala b/...egression/spark321/LinearRegression.scala → ...egression/spark322/LinearRegression.scala
@@ -17,7 +17,7 @@
  */
 // scalastyle:on
 
-package org.apache.spark.ml.regression.spark321
+package org.apache.spark.ml.regression.spark322
 
 import breeze.linalg.{DenseVector => BDV}
 import breeze.optimize.{

diff --git a/...sion/spark321/RandomForestRegressor.scala → ...sion/spark322/RandomForestRegressor.scala b/...sion/spark321/RandomForestRegressor.scala → ...sion/spark322/RandomForestRegressor.scala
@@ -17,7 +17,7 @@
  */
 // scalastyle:on
 
-package org.apache.spark.ml.regression.spark321
+package org.apache.spark.ml.regression.spark322
 
 import com.intel.oap.mllib.Utils
 import com.intel.oap.mllib.classification.{LearningNode => LearningNodeDAL}

diff --git a/.../spark/ml/stat/spark321/Correlation.scala → .../spark/ml/stat/spark322/Correlation.scala b/.../spark/ml/stat/spark321/Correlation.scala → .../spark/ml/stat/spark322/Correlation.scala
@@ -17,7 +17,7 @@
  */
 // scalastyle:on
 
-package org.apache.spark.ml.stat.spark321
+package org.apache.spark.ml.stat.spark322
 
 import com.intel.oap.mllib.Utils
 import com.intel.oap.mllib.stat.{CorrelationDALImpl, CorrelationShim}

diff --git a/...park/mllib/stat/spark321/Statistics.scala → ...park/mllib/stat/spark322/Statistics.scala b/...park/mllib/stat/spark321/Statistics.scala → ...park/mllib/stat/spark322/Statistics.scala
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.apache.spark.mllib.stat.spark321
+package org.apache.spark.mllib.stat.spark322
 
 import com.intel.oap.mllib.Utils
 import com.intel.oap.mllib.stat.{SummarizerDALImpl, SummarizerShim}