Modified code style

jeanlyn · May 14, 2015 · 7ae328f · 7ae328f
1 parent d5f794a
commit 7ae328f
Show file tree

Hide file tree

Showing 60 changed files with 570 additions and 361 deletions.
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -226,7 +226,9 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
       val accumulableHeaders: Seq[String] = Seq("Accumulable", "Value")
       def accumulableRow(acc: AccumulableInfo): Elem =
         <tr><td>{acc.name}</td><td>{acc.value}</td></tr>
-      val accumulableTable = UIUtils.listingTable(accumulableHeaders, accumulableRow,
+      val accumulableTable = UIUtils.listingTable(
+        accumulableHeaders,
+        accumulableRow,
         accumulables.values.toSeq)
 
       val taskHeadersAndCssClasses: Seq[(String, String)] =
@@ -265,9 +267,14 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
       val currentTime = System.currentTimeMillis()
       val taskTable = UIUtils.listingTable(
         unzipped._1,
-        taskRow(hasAccumulators, stageData.hasInput, stageData.hasOutput,
-          stageData.hasShuffleRead, stageData.hasShuffleWrite,
-          stageData.hasBytesSpilled, currentTime),
+        taskRow(
+          hasAccumulators,
+          stageData.hasInput,
+          stageData.hasOutput,
+          stageData.hasShuffleRead,
+          stageData.hasShuffleWrite,
+          stageData.hasBytesSpilled,
+          currentTime),
         tasks,
         headerClasses = unzipped._2)
       // Excludes tasks which failed and have incomplete metrics
@@ -515,49 +522,42 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
         val host = taskInfo.host
         executorsSet += ((executorId, host))
 
-        val isSucceeded = taskInfo.successful
-        val isFailed = taskInfo.failed
-        val isRunning = taskInfo.running
         val classNameByStatus = {
-          if (isSucceeded) {
+          if (taskInfo.successful) {
             "succeeded"
-          } else if (isFailed) {
+          } else if (taskInfo.failed) {
             "failed"
-          } else if (isRunning) {
+          } else if (taskInfo.running) {
             "running"
           }
         }
 
         val launchTime = taskInfo.launchTime
-        val finishTime = if (!isRunning) taskInfo.finishTime else currentTime
+        val finishTime = if (!taskInfo.running) taskInfo.finishTime else currentTime
         val totalExecutionTime = finishTime - launchTime
         minLaunchTime = launchTime.min(minLaunchTime)
         maxFinishTime = launchTime.max(maxFinishTime)
         numEffectiveTasks += 1
 
+        def toProportion(time: Long) = (time.toDouble / totalExecutionTime * 100).toLong
+
         val metricsOpt = taskUIData.taskMetrics
         val shuffleReadTime =
           metricsOpt.flatMap(_.shuffleReadMetrics.map(_.fetchWaitTime)).getOrElse(0L)
-        val shuffleReadTimeProportion =
-          (shuffleReadTime.toDouble / totalExecutionTime * 100).toLong
+        val shuffleReadTimeProportion = toProportion(shuffleReadTime)
         val shuffleWriteTime =
           (metricsOpt.flatMap(_.shuffleWriteMetrics
             .map(_.shuffleWriteTime)).getOrElse(0L) / 1e6).toLong
-        val shuffleWriteTimeProportion =
-          (shuffleWriteTime.toDouble / totalExecutionTime * 100).toLong
+        val shuffleWriteTimeProportion = toProportion(shuffleWriteTime)
         val executorComputingTime = metricsOpt.map(_.executorRunTime).getOrElse(0L) -
           shuffleReadTime - shuffleWriteTime
-        val executorComputingTimeProportion =
-          (executorComputingTime.toDouble / totalExecutionTime * 100).toLong
+        val executorComputingTimeProportion = toProportion(executorComputingTime)
         val serializationTime = metricsOpt.map(_.resultSerializationTime).getOrElse(0L)
-        val serializationTimeProportion =
-          (serializationTime.toDouble / totalExecutionTime * 100).toLong
+        val serializationTimeProportion = toProportion(serializationTime)
         val deserializationTime = metricsOpt.map(_.executorDeserializeTime).getOrElse(0L)
-        val deserializationTimeProportion =
-          (deserializationTime.toDouble / totalExecutionTime * 100).toLong
+        val deserializationTimeProportion = toProportion(deserializationTime)
         val gettingResultTime = getGettingResultTime(taskUIData.taskInfo)
-        val gettingResultTimeProportion =
-          (gettingResultTime.toDouble / totalExecutionTime * 100).toLong
+        val gettingResultTimeProportion = toProportion(gettingResultTime)
         val schedulerDelay = totalExecutionTime -
           (executorComputingTime + shuffleReadTime + shuffleWriteTime +
             serializationTime + deserializationTime + gettingResultTime)
@@ -585,22 +585,22 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
         val timelineObject =
           s"""
              {
-               'className': 'task task-assignment-timeline-object ${classNameByStatus}',
-               'group': '${executorId}',
+               'className': 'task task-assignment-timeline-object $classNameByStatus',
+               'group': '$executorId',
                'content': '<div class="task-assignment-timeline-content"' +
                  'data-toggle="tooltip" data-placement="top"' +
                  'data-html="true" data-container="body"' +
                  'data-title="${s"Task " + index + " (attempt " + attempt + ")"}<br>' +
                  'Status: ${taskInfo.status}<br>' +
                  'Launch Time: ${UIUtils.formatDate(new Date(launchTime))}' +
                  '${
-                     if (!isRunning) {
+                     if (!taskInfo.running) {
                        s"""<br>Finish Time: ${UIUtils.formatDate(new Date(finishTime))}"""
                      } else {
                         ""
                       }
                    }' +
-                 '<br>Scheduler Delay: ${schedulerDelay} ms' +
+                 '<br>Scheduler Delay: $schedulerDelay ms' +
                  '<br>Task Deserialization Time: ${UIUtils.formatDuration(deserializationTime)}' +
                  '<br>Shuffle Read Time: ${UIUtils.formatDuration(shuffleReadTime)}' +
                  '<br>Executor Computing Time: ${UIUtils.formatDuration(executorComputingTime)}' +
@@ -609,28 +609,28 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
                  '<br>Getting Result Time: ${UIUtils.formatDuration(gettingResultTime)}">' +
                  '<svg class="task-assignment-timeline-duration-bar">' +
                  '<rect class="scheduler-delay-proportion" ' +
-                   'x="${schedulerDelayProportionPos}%" y="0px" height="26px"' +
-                   'width="${schedulerDelayProportion}%""></rect>' +
+                   'x="$schedulerDelayProportionPos%" y="0px" height="26px"' +
+                   'width="$schedulerDelayProportion%""></rect>' +
                  '<rect class="deserialization-time-proportion" '+
-                   'x="${deserializationTimeProportionPos}%" y="0px" height="26px"' +
-                   'width="${deserializationTimeProportion}%"></rect>' +
+                   'x="$deserializationTimeProportionPos%" y="0px" height="26px"' +
+                   'width="$deserializationTimeProportion%"></rect>' +
                  '<rect class="shuffle-read-time-proportion" ' +
-                   'x="${shuffleReadTimeProportionPos}%" y="0px" height="26px"' +
-                   'width="${shuffleReadTimeProportion}%"></rect>' +
+                   'x="$shuffleReadTimeProportionPos%" y="0px" height="26px"' +
+                   'width="$shuffleReadTimeProportion%"></rect>' +
                  '<rect class="executor-runtime-proportion" ' +
-                   'x="${executorRuntimeProportionPos}%" y="0px" height="26px"' +
-                   'width="${executorComputingTimeProportion}%"></rect>' +
+                   'x="$executorRuntimeProportionPos%" y="0px" height="26px"' +
+                   'width="$executorComputingTimeProportion%"></rect>' +
                  '<rect class="shuffle-write-time-proportion" ' +
-                   'x="${shuffleWriteTimeProportionPos}%" y="0px" height="26px"' +
-                   'width="${shuffleWriteTimeProportion}%"></rect>' +
+                   'x="$shuffleWriteTimeProportionPos%" y="0px" height="26px"' +
+                   'width="$shuffleWriteTimeProportion%"></rect>' +
                  '<rect class="serialization-time-proportion" ' +
-                   'x="${serializationTimeProportionPos}%" y="0px" height="26px"' +
-                   'width="${serializationTimeProportion}%"></rect>' +
+                   'x="$serializationTimeProportionPos%" y="0px" height="26px"' +
+                   'width="$serializationTimeProportion%"></rect>' +
                  '<rect class="getting-result-time-proportion" ' +
-                   'x="${gettingResultTimeProportionPos}%" y="0px" height="26px"' +
-                   'width="${gettingResultTimeProportion}%"></rect></svg>',
-               'start': new Date(${launchTime}),
-               'end': new Date(${finishTime})
+                   'x="$gettingResultTimeProportionPos%" y="0px" height="26px"' +
+                   'width="$gettingResultTimeProportion%"></rect></svg>',
+               'start': new Date($launchTime),
+               'end': new Date($finishTime)
              }
            """
         timelineObject
@@ -640,23 +640,29 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
       case (executorId, host) =>
         s"""
             {
-              'id': '${executorId}',
-              'content': '${executorId} / ${host}',
+              'id': '$executorId',
+              'content': '$executorId / $host',
             }
           """
     }.mkString("[", ",", "]")
 
     val maxZoom = maxFinishTime - minLaunchTime
     <span class="expand-task-assignment-timeline">
       <span class="expand-task-assignment-timeline-arrow arrow-closed"></span>
-      <a>Event Timeline {
-        if (MAX_TIMELINE_TASKS <= numEffectiveTasks) {
-          s"(Most recent ${MAX_TIMELINE_TASKS})"
-        }
-      }
-      </a>
+      <a>Event Timeline</a>
     </span> ++
     <div id="task-assignment-timeline" class="collapsed">
+      {
+        if (MAX_TIMELINE_TASKS < numEffectiveTasks) {
+          <strong>
+            This stage has more than the maximum number of tasks that can be shown in the
+            visualization! Only the first {MAX_TIMELINE_TASKS} tasks
+            (of {numEffectiveTasks} total) are shown.
+          </strong>
+        } else {
+          Seq.empty
+        }
+      }
       <div class="control-panel">
         <div id="task-assignment-timeline-zoom-lock">
           <input type="checkbox"></input>
@@ -667,7 +673,7 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
     </div> ++
     <script type="text/javascript">
       {Unparsed(s"drawTaskAssignmentTimeline(" +
-      s"${groupArrayStr}, ${executorsArrayStr}, ${minLaunchTime}, ${maxZoom})")}
+      s"$groupArrayStr, $executorsArrayStr, $minLaunchTime, $maxZoom)")}
     </script>
   }
 

diff --git a/dev/create-release/create-release.sh b/dev/create-release/create-release.sh
@@ -118,14 +118,14 @@ if [[ ! "$@" =~ --skip-publish ]]; then
 
   rm -rf $SPARK_REPO
 
-  build/mvn -DskipTests -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
-    -Pyarn -Phive -Phive-thriftserver -Phadoop-2.2 -Pspark-ganglia-lgpl -Pkinesis-asl \
+  build/mvn -DskipTests -Pyarn -Phive \
+    -Phive-thriftserver -Phadoop-2.2 -Pspark-ganglia-lgpl -Pkinesis-asl \
     clean install
 
   ./dev/change-version-to-2.11.sh
 
-  build/mvn -DskipTests -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
-    -Dscala-2.11 -Pyarn -Phive -Phadoop-2.2 -Pspark-ganglia-lgpl -Pkinesis-asl \
+  build/mvn -DskipTests -Pyarn -Phive \
+    -Dscala-2.11 -Phadoop-2.2 -Pspark-ganglia-lgpl -Pkinesis-asl \
     clean install
 
   ./dev/change-version-to-2.10.sh
@@ -228,9 +228,9 @@ if [[ ! "$@" =~ --skip-package ]]; then
 
   # We increment the Zinc port each time to avoid OOM's and other craziness if multiple builds
   # share the same Zinc server.
-  make_binary_release "hadoop1" "-Phive -Phive-thriftserver -Dhadoop.version=1.0.4" "3030" &
-  make_binary_release "hadoop1-scala2.11" "-Phive -Dscala-2.11" "3031" &
-  make_binary_release "cdh4" "-Phive -Phive-thriftserver -Dhadoop.version=2.0.0-mr1-cdh4.2.0" "3032" &
+  make_binary_release "hadoop1" "-Phadoop-1 -Phive -Phive-thriftserver" "3030" &
+  make_binary_release "hadoop1-scala2.11" "-Phadoop-1 -Phive -Dscala-2.11" "3031" &
+  make_binary_release "cdh4" "-Phadoop-1 -Phive -Phive-thriftserver -Dhadoop.version=2.0.0-mr1-cdh4.2.0" "3032" &
   make_binary_release "hadoop2.3" "-Phadoop-2.3 -Phive -Phive-thriftserver -Pyarn" "3033" &
   make_binary_release "hadoop2.4" "-Phadoop-2.4 -Phive -Phive-thriftserver -Pyarn" "3034" &
   make_binary_release "mapr3" "-Pmapr3 -Phive -Phive-thriftserver" "3035" &

diff --git a/dev/run-tests b/dev/run-tests
@@ -40,11 +40,11 @@ function handle_error () {
 {
   if [ -n "$AMPLAB_JENKINS_BUILD_PROFILE" ]; then
     if [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop1.0" ]; then
-      export SBT_MAVEN_PROFILES_ARGS="-Dhadoop.version=1.0.4"
+      export SBT_MAVEN_PROFILES_ARGS="-Phadoop-1 -Dhadoop.version=1.0.4"
     elif [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop2.0" ]; then
-      export SBT_MAVEN_PROFILES_ARGS="-Dhadoop.version=2.0.0-mr1-cdh4.1.1"
+      export SBT_MAVEN_PROFILES_ARGS="-Phadoop-1 -Dhadoop.version=2.0.0-mr1-cdh4.1.1"
     elif [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop2.2" ]; then
-      export SBT_MAVEN_PROFILES_ARGS="-Pyarn -Phadoop-2.2 -Dhadoop.version=2.2.0"
+      export SBT_MAVEN_PROFILES_ARGS="-Pyarn -Phadoop-2.2"
     elif [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop2.3" ]; then
       export SBT_MAVEN_PROFILES_ARGS="-Pyarn -Phadoop-2.3 -Dhadoop.version=2.3.0"
     fi

diff --git a/dev/scalastyle b/dev/scalastyle
@@ -20,8 +20,8 @@
 echo -e "q\n" | build/sbt -Phive -Phive-thriftserver scalastyle > scalastyle.txt
 echo -e "q\n" | build/sbt -Phive -Phive-thriftserver test:scalastyle >> scalastyle.txt
 # Check style with YARN built too
-echo -e "q\n" | build/sbt -Pyarn -Phadoop-2.2 -Dhadoop.version=2.2.0 scalastyle >> scalastyle.txt
-echo -e "q\n" | build/sbt -Pyarn -Phadoop-2.2 -Dhadoop.version=2.2.0 test:scalastyle >> scalastyle.txt
+echo -e "q\n" | build/sbt -Pyarn -Phadoop-2.2 scalastyle >> scalastyle.txt
+echo -e "q\n" | build/sbt -Pyarn -Phadoop-2.2 test:scalastyle >> scalastyle.txt
 
 ERRORS=$(cat scalastyle.txt | awk '{if($1~/error/)print}')
 rm scalastyle.txt

diff --git a/docs/building-spark.md b/docs/building-spark.md
@@ -59,14 +59,14 @@ You can fix this by setting the `MAVEN_OPTS` variable as discussed before.
 
 # Specifying the Hadoop Version
 
-Because HDFS is not protocol-compatible across versions, if you want to read from HDFS, you'll need to build Spark against the specific HDFS version in your environment. You can do this through the "hadoop.version" property. If unset, Spark will build against Hadoop 1.0.4 by default. Note that certain build profiles are required for particular Hadoop versions:
+Because HDFS is not protocol-compatible across versions, if you want to read from HDFS, you'll need to build Spark against the specific HDFS version in your environment. You can do this through the "hadoop.version" property. If unset, Spark will build against Hadoop 2.2.0 by default. Note that certain build profiles are required for particular Hadoop versions:
 
 <table class="table">
   <thead>
     <tr><th>Hadoop version</th><th>Profile required</th></tr>
   </thead>
   <tbody>
-    <tr><td>1.x to 2.1.x</td><td>(none)</td></tr>
+    <tr><td>1.x to 2.1.x</td><td>hadoop-1</td></tr>
     <tr><td>2.2.x</td><td>hadoop-2.2</td></tr>
     <tr><td>2.3.x</td><td>hadoop-2.3</td></tr>
     <tr><td>2.4.x</td><td>hadoop-2.4</td></tr>
@@ -77,19 +77,20 @@ For Apache Hadoop versions 1.x, Cloudera CDH "mr1" distributions, and other Hado
 
 {% highlight bash %}
 # Apache Hadoop 1.2.1
-mvn -Dhadoop.version=1.2.1 -DskipTests clean package
+mvn -Dhadoop.version=1.2.1 -Phadoop-1 -DskipTests clean package
 
 # Cloudera CDH 4.2.0 with MapReduce v1
-mvn -Dhadoop.version=2.0.0-mr1-cdh4.2.0 -DskipTests clean package
+mvn -Dhadoop.version=2.0.0-mr1-cdh4.2.0 -Phadoop-1 -DskipTests clean package
 {% endhighlight %}
 
 You can enable the "yarn" profile and optionally set the "yarn.version" property if it is different from "hadoop.version". Spark only supports YARN versions 2.2.0 and later.
 
 Examples:
 
 {% highlight bash %}
+
 # Apache Hadoop 2.2.X
-mvn -Pyarn -Phadoop-2.2 -Dhadoop.version=2.2.0 -DskipTests clean package
+mvn -Pyarn -Phadoop-2.2 -DskipTests clean package
 
 # Apache Hadoop 2.3.X
 mvn -Pyarn -Phadoop-2.3 -Dhadoop.version=2.3.0 -DskipTests clean package

diff --git a/docs/hadoop-third-party-distributions.md b/docs/hadoop-third-party-distributions.md
@@ -14,7 +14,7 @@ property. For certain versions, you will need to specify additional profiles. Fo
 see the guide on [building with maven](building-spark.html#specifying-the-hadoop-version):
 
     mvn -Dhadoop.version=1.0.4 -DskipTests clean package
-    mvn -Phadoop-2.2 -Dhadoop.version=2.2.0 -DskipTests clean package
+    mvn -Phadoop-2.3 -Dhadoop.version=2.3.0 -DskipTests clean package
 
 The table below lists the corresponding `hadoop.version` code for each CDH/HDP release. Note that
 some Hadoop releases are binary compatible across client versions. This means the pre-built Spark