From 1561cecc0ac5f64d14c0b4cf33b882f39c3d32bf Mon Sep 17 00:00:00 2001 From: Sean Owen Date: Thu, 27 Feb 2014 10:11:10 +0000 Subject: [PATCH 1/2] Remove "exclude *" dependencies that are causing Maven warnings, and that are apparently unneeded anyway --- examples/pom.xml | 8 -------- external/flume/pom.xml | 4 ---- pom.xml | 34 +--------------------------------- project/SparkBuild.scala | 20 +++++++++----------- streaming/pom.xml | 5 ----- 5 files changed, 10 insertions(+), 61 deletions(-) diff --git a/examples/pom.xml b/examples/pom.xml index 709907cb7ed87..d952e2ca4e5dc 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -169,14 +169,6 @@ org.apache.cassandra.deps avro - - org.sonatype.sisu.inject - * - - - org.xerial.snappy - * - diff --git a/external/flume/pom.xml b/external/flume/pom.xml index 95ff872196dd0..bc00ab4573c39 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -53,10 +53,6 @@ org.jboss.netty netty - - org.xerial.snappy - * - diff --git a/pom.xml b/pom.xml index 39995396b38f4..ef60548f19475 100644 --- a/pom.xml +++ b/pom.xml @@ -374,7 +374,7 @@ 3.1 test - + org.mockito mockito-all 1.8.5 @@ -416,14 +416,6 @@ org.jboss.netty netty - - org.codehaus.jackson - * - - - org.sonatype.sisu.inject - * - commons-logging commons-logging @@ -448,14 +440,6 @@ org.jboss.netty netty - - org.codehaus.jackson - * - - - org.sonatype.sisu.inject - * - @@ -471,14 +455,6 @@ org.jboss.netty netty - - org.codehaus.jackson - * - - - org.sonatype.sisu.inject - * - @@ -495,14 +471,6 @@ org.jboss.netty netty - - org.codehaus.jackson - * - - - org.sonatype.sisu.inject - * - diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 399886ca1b821..d45f6773fad78 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -243,13 +243,11 @@ object SparkBuild extends Build { val slf4jVersion = "1.7.5" - val excludeCglib = ExclusionRule(organization = "org.sonatype.sisu.inject") - val excludeJackson = ExclusionRule(organization = "org.codehaus.jackson") val excludeNetty = ExclusionRule(organization = "org.jboss.netty") val excludeAsm = ExclusionRule(organization = "asm") - val excludeSnappy = ExclusionRule(organization = "org.xerial.snappy") val excludeCommonsLogging = ExclusionRule(organization = "commons-logging") val excludeSLF4J = ExclusionRule(organization = "org.slf4j") + val excludeScalap = ExclusionRule(organization = "org.scala-lang", artifact = "scalap") def coreSettings = sharedSettings ++ Seq( name := "spark-core", @@ -273,13 +271,13 @@ object SparkBuild extends Build { "org.spark-project.akka" %% "akka-remote" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty), "org.spark-project.akka" %% "akka-slf4j" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty), "org.spark-project.akka" %% "akka-testkit" % "2.2.3-shaded-protobuf" % "test", - "org.json4s" %% "json4s-jackson" % "3.2.6", + "org.json4s" %% "json4s-jackson" % "3.2.6" excludeAll(excludeScalap), "it.unimi.dsi" % "fastutil" % "6.4.4", "colt" % "colt" % "1.2.0", "org.apache.mesos" % "mesos" % "0.13.0", "net.java.dev.jets3t" % "jets3t" % "0.7.1" excludeAll(excludeCommonsLogging), "org.apache.derby" % "derby" % "10.4.2.0" % "test", - "org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib, excludeCommonsLogging, excludeSLF4J), + "org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeCommonsLogging, excludeSLF4J), "org.apache.curator" % "curator-recipes" % "2.4.0" excludeAll(excludeNetty), "com.codahale.metrics" % "metrics-core" % "3.0.0", "com.codahale.metrics" % "metrics-jvm" % "3.0.0", @@ -316,7 +314,7 @@ object SparkBuild extends Build { exclude("io.netty", "netty") exclude("jline","jline") exclude("org.apache.cassandra.deps", "avro") - excludeAll(excludeSnappy, excludeCglib, excludeSLF4J) + excludeAll(excludeSLF4J) ) ) ++ assemblySettings ++ extraAssemblySettings @@ -379,10 +377,10 @@ object SparkBuild extends Build { def yarnEnabledSettings = Seq( libraryDependencies ++= Seq( // Exclude rule required for all ? - "org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib), - "org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib), - "org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib), - "org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib) + "org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm), + "org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion excludeAll(excludeNetty, excludeAsm), + "org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion excludeAll(excludeNetty, excludeAsm), + "org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion excludeAll(excludeNetty, excludeAsm) ) ) @@ -428,7 +426,7 @@ object SparkBuild extends Build { def flumeSettings() = sharedSettings ++ Seq( name := "spark-streaming-flume", libraryDependencies ++= Seq( - "org.apache.flume" % "flume-ng-sdk" % "1.2.0" % "compile" excludeAll(excludeNetty, excludeSnappy) + "org.apache.flume" % "flume-ng-sdk" % "1.2.0" % "compile" excludeAll(excludeNetty) ) ) diff --git a/streaming/pom.xml b/streaming/pom.xml index 2f358c99380a6..1f3366e25f6a9 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -55,11 +55,6 @@ org.eclipse.jetty jetty-server - - org.codehaus.jackson - jackson-mapper-asl - 1.9.11 - org.scala-lang scala-library From 9240abd9e2870ef6ad6a3781d37087501e4aedd9 Mon Sep 17 00:00:00 2001 From: Sean Owen Date: Thu, 27 Feb 2014 11:49:46 +0000 Subject: [PATCH 2/2] Avoid scala version conflict in scalap induced by json4s dependency --- core/pom.xml | 9 +++++++++ pom.xml | 3 +++ 2 files changed, 12 insertions(+) diff --git a/core/pom.xml b/core/pom.xml index 6626d7c1b96b5..2afd25082534e 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -125,6 +125,15 @@ org.json4s json4s-jackson_${scala.binary.version} 3.2.6 + + + + org.scala-lang + scalap + + it.unimi.dsi diff --git a/pom.xml b/pom.xml index ef60548f19475..7e04efaa9cefc 100644 --- a/pom.xml +++ b/pom.xml @@ -310,6 +310,9 @@ net.liftweb lift-json_${scala.binary.version} 2.5.1 + org.scala-lang