From 969bb60cfc2f41d3c81a9d4af06055260669cc04 Mon Sep 17 00:00:00 2001
From: Chen Haifeng <haifeng.chen@intel.com>
Date: Thu, 10 Jun 2021 06:21:45 +0800
Subject: [PATCH] [ML-76] Reorganize multiple Spark version support code
 structure

---
 mllib-dal/pom.xml                             | 109 ++++++++----------
 .../spark}/ml/classification/NaiveBayes.scala |   0
 .../apache/spark}/ml/clustering/KMeans.scala  |   0
 .../org/apache/spark}/ml/feature/PCA.scala    |   0
 .../apache/spark}/ml/recommendation/ALS.scala |   0
 .../spark}/mllib/clustering/KMeans.scala      |   0
 .../apache/spark}/ml/clustering/KMeans.scala  |   0
 .../org/apache/spark}/ml/feature/PCA.scala    |   0
 .../apache/spark}/ml/recommendation/ALS.scala |   0
 .../spark}/mllib/clustering/KMeans.scala      |   0
 .../apache/spark}/ml/clustering/KMeans.scala  |   0
 .../org/apache/spark}/ml/feature/PCA.scala    |   0
 .../apache/spark}/ml/recommendation/ALS.scala |   0
 .../spark}/mllib/clustering/KMeans.scala      |   0
 .../apache/spark}/ml/clustering/KMeans.scala  |   0
 .../org/apache/spark}/ml/feature/PCA.scala    |   0
 .../apache/spark}/ml/recommendation/ALS.scala |   0
 .../spark}/mllib/clustering/KMeans.scala      |   0
 18 files changed, 48 insertions(+), 61 deletions(-)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.0.0 => spark-3.0.0/main/scala/org/apache/spark}/ml/classification/NaiveBayes.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.0.0 => spark-3.0.0/main/scala/org/apache/spark}/ml/clustering/KMeans.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.0.0 => spark-3.0.0/main/scala/org/apache/spark}/ml/feature/PCA.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.0.0 => spark-3.0.0/main/scala/org/apache/spark}/ml/recommendation/ALS.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.0.0 => spark-3.0.0/main/scala/org/apache/spark}/mllib/clustering/KMeans.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.0.1 => spark-3.0.1/main/scala/org/apache/spark}/ml/clustering/KMeans.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.0.1 => spark-3.0.1/main/scala/org/apache/spark}/ml/feature/PCA.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.0.1 => spark-3.0.1/main/scala/org/apache/spark}/ml/recommendation/ALS.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.0.1 => spark-3.0.1/main/scala/org/apache/spark}/mllib/clustering/KMeans.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.0.2 => spark-3.0.2/main/scala/org/apache/spark}/ml/clustering/KMeans.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.0.2 => spark-3.0.2/main/scala/org/apache/spark}/ml/feature/PCA.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.0.2 => spark-3.0.2/main/scala/org/apache/spark}/ml/recommendation/ALS.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.0.2 => spark-3.0.2/main/scala/org/apache/spark}/mllib/clustering/KMeans.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.1.1 => spark-3.1.1/main/scala/org/apache/spark}/ml/clustering/KMeans.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.1.1 => spark-3.1.1/main/scala/org/apache/spark}/ml/feature/PCA.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.1.1 => spark-3.1.1/main/scala/org/apache/spark}/ml/recommendation/ALS.scala (100%)
 rename mllib-dal/src/{main/scala/org/apache/spark-3.1.1 => spark-3.1.1/main/scala/org/apache/spark}/mllib/clustering/KMeans.scala (100%)

diff --git a/mllib-dal/pom.xml b/mllib-dal/pom.xml
index d37ff879c..d2594e4f7 100644
--- a/mllib-dal/pom.xml
+++ b/mllib-dal/pom.xml
@@ -157,16 +157,6 @@
                 <spark.version>3.0.0</spark.version>
                 <scalatest.version>3.0.8</scalatest.version>
             </properties>
-            <dependencyManagement>
-                <dependencies>
-                </dependencies>
-            </dependencyManagement>
-            <build>
-                <pluginManagement>
-                    <plugins>
-                    </plugins>
-                </pluginManagement>
-            </build>
         </profile>
 
         <profile>
@@ -175,16 +165,6 @@
                 <spark.version>3.0.1</spark.version>
                 <scalatest.version>3.0.8</scalatest.version>
             </properties>
-            <dependencyManagement>
-                <dependencies>
-                </dependencies>
-            </dependencyManagement>
-            <build>
-                <pluginManagement>
-                    <plugins>
-                    </plugins>
-                </pluginManagement>
-            </build>
         </profile>
 
         <profile>
@@ -193,16 +173,6 @@
                 <spark.version>3.0.2</spark.version>
                 <scalatest.version>3.0.8</scalatest.version>
             </properties>
-            <dependencyManagement>
-                <dependencies>
-                </dependencies>
-            </dependencyManagement>
-            <build>
-                <pluginManagement>
-                    <plugins>
-                    </plugins>
-                </pluginManagement>
-            </build>
         </profile>
 
         <profile>
@@ -211,46 +181,63 @@
                 <spark.version>3.1.1</spark.version>
                 <scalatest.version>3.2.3</scalatest.version>
             </properties>
-            <dependencyManagement>
-                <dependencies>
-                </dependencies>
-            </dependencyManagement>
-            <build>
-                <pluginManagement>
-                    <plugins>
-                    </plugins>
-                </pluginManagement>
-            </build>
         </profile>
     </profiles>
 
     <build>
         <plugins>
-
-            <plugin>
-                <groupId>org.scala-tools</groupId>
-                <artifactId>maven-scala-plugin</artifactId>
-                <version>2.15.2</version>
-                <executions>
+                <plugin>
+                  <groupId>org.codehaus.mojo</groupId>
+                  <artifactId>build-helper-maven-plugin</artifactId>
+                  <version>3.2.0</version>
+                  <executions>
                     <execution>
-                        <id>compile</id>
-                        <goals>
-                            <goal>compile</goal>
-                        </goals>
-                        <configuration>
-                            <includes>
-                                <include>**/*.java</include>
-                                <include>**/spark/**</include>
-                                <include>**/spark-${spark.version}/**</include>
-                            </includes>
-                        </configuration>
+                      <id>add-source</id>
+                      <phase>generate-sources</phase>
+                      <goals>
+                        <goal>add-source</goal>
+                      </goals>
+                      <configuration>
+                        <sources>
+                          <source>src/spark-${spark.version}/main/java</source>
+                          <source>src/spark-${spark.version}/main/scala</source>
+                        </sources>
+                      </configuration>
                     </execution>
                     <execution>
-                        <id>testCompile</id>
-                        <goals>
-                            <goal>testCompile</goal>
-                        </goals>
+                      <id>add-test-source</id>
+                      <phase>generate-sources</phase>
+                      <goals>
+                        <goal>add-test-source</goal>
+                      </goals>
+                      <configuration>
+                        <sources>
+                          <source>src/spark-${spark.version}/test/scala</source>
+                        </sources>
+                      </configuration>
                     </execution>
+                  </executions>
+                </plugin>
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+                <version>4.4.0</version>
+                <executions>
+                  <execution>
+                    <id>scala-compile-first</id>
+                    <phase>process-resources</phase>
+                    <goals>
+                      <goal>add-source</goal>
+                      <goal>compile</goal>
+                    </goals>
+                  </execution>
+                  <execution>
+                    <id>scala-test-compile</id>
+                    <phase>process-test-resources</phase>
+                    <goals>
+                      <goal>testCompile</goal>
+                    </goals>
+                  </execution>
                 </executions>
                 <configuration>
                     <scalaVersion>${scala.version}</scalaVersion>
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.0.0/ml/classification/NaiveBayes.scala b/mllib-dal/src/spark-3.0.0/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.0.0/ml/classification/NaiveBayes.scala
rename to mllib-dal/src/spark-3.0.0/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.0.0/ml/clustering/KMeans.scala b/mllib-dal/src/spark-3.0.0/main/scala/org/apache/spark/ml/clustering/KMeans.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.0.0/ml/clustering/KMeans.scala
rename to mllib-dal/src/spark-3.0.0/main/scala/org/apache/spark/ml/clustering/KMeans.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.0.0/ml/feature/PCA.scala b/mllib-dal/src/spark-3.0.0/main/scala/org/apache/spark/ml/feature/PCA.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.0.0/ml/feature/PCA.scala
rename to mllib-dal/src/spark-3.0.0/main/scala/org/apache/spark/ml/feature/PCA.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.0.0/ml/recommendation/ALS.scala b/mllib-dal/src/spark-3.0.0/main/scala/org/apache/spark/ml/recommendation/ALS.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.0.0/ml/recommendation/ALS.scala
rename to mllib-dal/src/spark-3.0.0/main/scala/org/apache/spark/ml/recommendation/ALS.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.0.0/mllib/clustering/KMeans.scala b/mllib-dal/src/spark-3.0.0/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.0.0/mllib/clustering/KMeans.scala
rename to mllib-dal/src/spark-3.0.0/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.0.1/ml/clustering/KMeans.scala b/mllib-dal/src/spark-3.0.1/main/scala/org/apache/spark/ml/clustering/KMeans.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.0.1/ml/clustering/KMeans.scala
rename to mllib-dal/src/spark-3.0.1/main/scala/org/apache/spark/ml/clustering/KMeans.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.0.1/ml/feature/PCA.scala b/mllib-dal/src/spark-3.0.1/main/scala/org/apache/spark/ml/feature/PCA.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.0.1/ml/feature/PCA.scala
rename to mllib-dal/src/spark-3.0.1/main/scala/org/apache/spark/ml/feature/PCA.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.0.1/ml/recommendation/ALS.scala b/mllib-dal/src/spark-3.0.1/main/scala/org/apache/spark/ml/recommendation/ALS.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.0.1/ml/recommendation/ALS.scala
rename to mllib-dal/src/spark-3.0.1/main/scala/org/apache/spark/ml/recommendation/ALS.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.0.1/mllib/clustering/KMeans.scala b/mllib-dal/src/spark-3.0.1/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.0.1/mllib/clustering/KMeans.scala
rename to mllib-dal/src/spark-3.0.1/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.0.2/ml/clustering/KMeans.scala b/mllib-dal/src/spark-3.0.2/main/scala/org/apache/spark/ml/clustering/KMeans.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.0.2/ml/clustering/KMeans.scala
rename to mllib-dal/src/spark-3.0.2/main/scala/org/apache/spark/ml/clustering/KMeans.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.0.2/ml/feature/PCA.scala b/mllib-dal/src/spark-3.0.2/main/scala/org/apache/spark/ml/feature/PCA.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.0.2/ml/feature/PCA.scala
rename to mllib-dal/src/spark-3.0.2/main/scala/org/apache/spark/ml/feature/PCA.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.0.2/ml/recommendation/ALS.scala b/mllib-dal/src/spark-3.0.2/main/scala/org/apache/spark/ml/recommendation/ALS.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.0.2/ml/recommendation/ALS.scala
rename to mllib-dal/src/spark-3.0.2/main/scala/org/apache/spark/ml/recommendation/ALS.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.0.2/mllib/clustering/KMeans.scala b/mllib-dal/src/spark-3.0.2/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.0.2/mllib/clustering/KMeans.scala
rename to mllib-dal/src/spark-3.0.2/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.1.1/ml/clustering/KMeans.scala b/mllib-dal/src/spark-3.1.1/main/scala/org/apache/spark/ml/clustering/KMeans.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.1.1/ml/clustering/KMeans.scala
rename to mllib-dal/src/spark-3.1.1/main/scala/org/apache/spark/ml/clustering/KMeans.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.1.1/ml/feature/PCA.scala b/mllib-dal/src/spark-3.1.1/main/scala/org/apache/spark/ml/feature/PCA.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.1.1/ml/feature/PCA.scala
rename to mllib-dal/src/spark-3.1.1/main/scala/org/apache/spark/ml/feature/PCA.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.1.1/ml/recommendation/ALS.scala b/mllib-dal/src/spark-3.1.1/main/scala/org/apache/spark/ml/recommendation/ALS.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.1.1/ml/recommendation/ALS.scala
rename to mllib-dal/src/spark-3.1.1/main/scala/org/apache/spark/ml/recommendation/ALS.scala
diff --git a/mllib-dal/src/main/scala/org/apache/spark-3.1.1/mllib/clustering/KMeans.scala b/mllib-dal/src/spark-3.1.1/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
similarity index 100%
rename from mllib-dal/src/main/scala/org/apache/spark-3.1.1/mllib/clustering/KMeans.scala
rename to mllib-dal/src/spark-3.1.1/main/scala/org/apache/spark/mllib/clustering/KMeans.scala