From 1ad5d3d20eba0156569f2ae9e605520fcaca471d Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Mon, 20 Aug 2018 13:54:45 -0700 Subject: [PATCH] Revert "[SPARK-24418][BUILD] Upgrade Scala to 2.11.12 and 2.12.6" This reverts commit c7967c6049327a03b63ea7a3b0001a97d31e309d. --- LICENSE | 2 +- dev/deps/spark-deps-hadoop-2.6 | 10 ++++---- dev/deps/spark-deps-hadoop-2.7 | 10 ++++---- dev/deps/spark-deps-hadoop-3.1 | 10 ++++---- pom.xml | 8 +++---- .../org/apache/spark/repl/SparkILoop.scala | 24 ++++++++++++------- .../spark/repl/SparkILoopInterpreter.scala | 18 ++------------ 7 files changed, 37 insertions(+), 45 deletions(-) diff --git a/LICENSE b/LICENSE index b771bd552b762..3514a1eb2d672 100644 --- a/LICENSE +++ b/LICENSE @@ -258,4 +258,4 @@ data/mllib/images/kittens/29.5.a_b_EGDP022204.jpg data/mllib/images/kittens/54893.jpg data/mllib/images/kittens/DP153539.jpg data/mllib/images/kittens/DP802813.jpg -data/mllib/images/multi-channel/chr30.4.184.jpg \ No newline at end of file +data/mllib/images/multi-channel/chr30.4.184.jpg diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index aca5bbbe02eb6..1d45d3750ec97 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -118,7 +118,7 @@ jersey-media-jaxb-2.22.2.jar jersey-server-2.22.2.jar jetty-6.1.26.jar jetty-util-6.1.26.jar -jline-2.14.6.jar +jline-2.12.1.jar joda-time-2.9.3.jar jodd-core-3.5.2.jar jpam-1.1.jar @@ -169,10 +169,10 @@ parquet-jackson-1.10.0.jar protobuf-java-2.5.0.jar py4j-0.10.7.jar pyrolite-4.13.jar -scala-compiler-2.11.12.jar -scala-library-2.11.12.jar -scala-parser-combinators_2.11-1.1.0.jar -scala-reflect-2.11.12.jar +scala-compiler-2.11.8.jar +scala-library-2.11.8.jar +scala-parser-combinators_2.11-1.0.4.jar +scala-reflect-2.11.8.jar scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar slf4j-api-1.7.16.jar diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index 0902af1dfdc65..1cc3c51b3e3f6 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -119,7 +119,7 @@ jersey-server-2.22.2.jar jetty-6.1.26.jar jetty-sslengine-6.1.26.jar jetty-util-6.1.26.jar -jline-2.14.6.jar +jline-2.12.1.jar joda-time-2.9.3.jar jodd-core-3.5.2.jar jpam-1.1.jar @@ -171,10 +171,10 @@ parquet-jackson-1.10.0.jar protobuf-java-2.5.0.jar py4j-0.10.7.jar pyrolite-4.13.jar -scala-compiler-2.11.12.jar -scala-library-2.11.12.jar -scala-parser-combinators_2.11-1.1.0.jar -scala-reflect-2.11.12.jar +scala-compiler-2.11.8.jar +scala-library-2.11.8.jar +scala-parser-combinators_2.11-1.0.4.jar +scala-reflect-2.11.8.jar scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar slf4j-api-1.7.16.jar diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1 index 35cf1cd85fd80..599b3a972bf42 100644 --- a/dev/deps/spark-deps-hadoop-3.1 +++ b/dev/deps/spark-deps-hadoop-3.1 @@ -118,7 +118,7 @@ jersey-media-jaxb-2.22.2.jar jersey-server-2.22.2.jar jetty-webapp-9.3.24.v20180605.jar jetty-xml-9.3.24.v20180605.jar -jline-2.14.6.jar +jline-2.12.1.jar joda-time-2.9.3.jar jodd-core-3.5.2.jar jpam-1.1.jar @@ -189,10 +189,10 @@ protobuf-java-2.5.0.jar py4j-0.10.7.jar pyrolite-4.13.jar re2j-1.1.jar -scala-compiler-2.11.12.jar -scala-library-2.11.12.jar -scala-parser-combinators_2.11-1.1.0.jar -scala-reflect-2.11.12.jar +scala-compiler-2.11.8.jar +scala-library-2.11.8.jar +scala-parser-combinators_2.11-1.0.4.jar +scala-reflect-2.11.8.jar scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar slf4j-api-1.7.16.jar diff --git a/pom.xml b/pom.xml index a4184c3153336..5eecd037993f4 100644 --- a/pom.xml +++ b/pom.xml @@ -155,7 +155,7 @@ 3.4.1 3.2.2 - 2.11.12 + 2.11.8 2.11 1.9.13 2.6.7 @@ -740,13 +740,13 @@ org.scala-lang.modules scala-parser-combinators_${scala.binary.version} - 1.1.0 + 1.0.4 jline jline - 2.14.6 + 2.12.1 org.scalatest @@ -2756,7 +2756,7 @@ scala-2.12 - 2.12.6 + 2.12.4 2.12 diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala index a44051b351e19..e69441a475e9a 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala @@ -36,7 +36,7 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) def this() = this(None, new JPrintWriter(Console.out, true)) override def createInterpreter(): Unit = { - intp = new SparkILoopInterpreter(settings, out, initializeSpark) + intp = new SparkILoopInterpreter(settings, out) } val initializationCommands: Seq[String] = Seq( @@ -73,15 +73,11 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) "import org.apache.spark.sql.functions._" ) - def initializeSpark(): Unit = { - if (!intp.reporter.hasErrors) { - // `savingReplayStack` removes the commands from session history. - savingReplayStack { - initializationCommands.foreach(intp quietRun _) + def initializeSpark() { + intp.beQuietDuring { + savingReplayStack { // remove the commands from session history. + initializationCommands.foreach(processLine) } - } else { - throw new RuntimeException(s"Scala $versionString interpreter encountered " + - "errors during initialization") } } @@ -105,6 +101,16 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) /** Available commands */ override def commands: List[LoopCommand] = standardCommands + /** + * We override `loadFiles` because we need to initialize Spark *before* the REPL + * sees any files, so that the Spark context is visible in those files. This is a bit of a + * hack, but there isn't another hook available to us at this point. + */ + override def loadFiles(settings: Settings): Unit = { + initializeSpark() + super.loadFiles(settings) + } + override def resetCommand(line: String): Unit = { super.resetCommand(line) initializeSpark() diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala index 4e63816402a10..e736607a9a6b9 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala @@ -21,22 +21,8 @@ import scala.collection.mutable import scala.tools.nsc.Settings import scala.tools.nsc.interpreter._ -class SparkILoopInterpreter(settings: Settings, out: JPrintWriter, initializeSpark: () => Unit) - extends IMain(settings, out) { self => - - /** - * We override `initializeSynchronous` to initialize Spark *after* `intp` is properly initialized - * and *before* the REPL sees any files in the private `loadInitFiles` functions, so that - * the Spark context is visible in those files. - * - * This is a bit of a hack, but there isn't another hook available to us at this point. - * - * See the discussion in Scala community https://github.com/scala/bug/issues/10913 for detail. - */ - override def initializeSynchronous(): Unit = { - super.initializeSynchronous() - initializeSpark() - } +class SparkILoopInterpreter(settings: Settings, out: JPrintWriter) extends IMain(settings, out) { + self => override lazy val memberHandlers = new { val intp: self.type = self