Skip to content

Commit

Permalink
Merge pull request apache#92 from shivaram/sparkr-yarn
Browse files Browse the repository at this point in the history 
[SPARKR-118] SparkR build update to work with YARN
  • Loading branch information
@concretevitamin
concretevitamin committed Oct 30, 2014
2 parents be82dcc + 5951d3b commit 8e8a029
Show file tree
Hide file tree
Showing 3 changed files with 193 additions and 5 deletions.
9 changes: 8 additions & 1 deletion pkg/src/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ RESOURCE_FILES := $(wildcard $(RESOURCE_DIR)/*)

SPARK_HADOOP_VERSION ?= 1.0.4
SPARK_VERSION ?= 1.1.0
SPARK_YARN_VERSION ?= 2.4.0

ifdef USE_MAVEN
TARGET_NAME := $(MAVEN_TARGET_NAME)
Expand All @@ -24,14 +25,20 @@ else
BUILD_TOOL := sbt/sbt
endif

ifdef USE_YARN
MAVEN_YARN_FLAG := "-Pyarn"
else
MAVEN_YARN_FLAG := ""
endif

all: $(TARGET_NAME)

$(SBT_TARGET_NAME): build.sbt $(SCALA_FILES) $(RESOURCE_FILES)
./sbt/sbt assembly
cp -f $(SBT_TARGET_NAME) ../inst/

$(MAVEN_TARGET_NAME): pom.xml $(SCALA_FILES) $(RESOURCE_FILES)
mvn -Dhadoop.version=$(SPARK_HADOOP_VERSION) -Dspark.version=$(SPARK_VERSION) -DskipTests clean package shade:shade
mvn -Dhadoop.version=$(SPARK_HADOOP_VERSION) -Dspark.version=$(SPARK_VERSION) -DskipTests $(MAVEN_YARN_FLAG) -Dyarn.version=$(SPARK_YARN_VERSION) clean package shade:shade
cp -f $(MAVEN_TARGET_NAME) ../inst/$(JAR_NAME)

clean:
Expand Down
20 changes: 18 additions & 2 deletions pkg/src/build.sbt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,28 @@ libraryDependencies ++= Seq(
val excludeNetty = ExclusionRule(organization = "org.jboss.netty")
val excludeAsm = ExclusionRule(organization = "asm")
val excludeSnappy = ExclusionRule(organization = "org.xerial.snappy")
val sbtYarnFlag = scala.util.Properties.envOrElse("USE_YARN", "")
val defaultHadoopVersion = "1.0.4"
val defaultSparkVersion = "1.1.0"
val hadoopVersion = scala.util.Properties.envOrElse("SPARK_HADOOP_VERSION", defaultHadoopVersion)
val sparkVersion = scala.util.Properties.envOrElse("SPARK_VERSION", defaultSparkVersion)
libraryDependencies += "org.apache.hadoop" % "hadoop-client" % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib)
libraryDependencies += "org.apache.spark" % "spark-core_2.10" % sparkVersion
libraryDependencies ++= Seq(
"org.apache.hadoop" % "hadoop-client" % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib),
"org.apache.spark" % "spark-core_2.10" % sparkVersion
) ++ (if (sbtYarnFlag != "") {
val defaultYarnVersion = "2.4.0"
val yarnVersion = scala.util.Properties.envOrElse("SPARK_YARN_VERSION", defaultYarnVersion)
Seq(
"org.apache.hadoop" % "hadoop-yarn-api" % yarnVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib),
"org.apache.hadoop" % "hadoop-yarn-common" % yarnVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib),
"org.apache.hadoop" % "hadoop-yarn-server-web-proxy" % yarnVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib),
"org.apache.hadoop" % "hadoop-yarn-client" % yarnVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib),
"org.apache.spark" % "spark-yarn_2.10" % sparkVersion
)
} else {
None.toSeq
}
)
}

resolvers ++= Seq(
Expand Down
169 changes: 167 additions & 2 deletions pkg/src/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,51 @@
<packaging>jar</packaging>
<version>0.1</version>
<repositories>
<repository>
<id>central</id>
<!-- This should be at top, it makes maven try the central repo first and then others and hence faster dep resolution -->
<name>Maven Repository</name>
<url>https://repo1.maven.org/maven2</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>apache-repo</id>
<name>Apache Repository</name>
<url>https://repository.apache.org/content/repositories/releases</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>jboss-repo</id>
<name>JBoss Repository</name>
<url>https://repository.jboss.org/nexus/content/repositories/releases</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>cloudera-repo</id>
<name>Cloudera Repository</name>
<url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>Spray.cc repository</id>
<url>http://repo.spray.cc</url>
Expand Down Expand Up @@ -73,6 +118,7 @@
<version>2.10.3</version>
</dependency>


</dependencies>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
Expand All @@ -82,8 +128,6 @@
<scala.version>2.10.3</scala.version>
<scala.binary.version>2.10</scala.binary.version>

<hadoop.version>1.0.4</hadoop.version>

<PermGen>64m</PermGen>
<MaxPermGen>512m</MaxPermGen>
</properties>
Expand Down Expand Up @@ -298,4 +342,125 @@
</pluginRepository>
</pluginRepositories>

<profiles>
<profile>
<id>yarn</id>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
<version>${yarn.version}</version>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
<version>${yarn.version}</version>
<exclusions>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
<version>${yarn.version}</version>
<exclusions>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
<version>${yarn.version}</version>
<exclusions>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_2.10</artifactId>
<version>${spark.version}</version>
</dependency>
</dependencies>
</profile>
</profiles>

</project>

0 comments on commit 8e8a029

Please sign in to comment.