Fix bugs for EnvironmentUI and ExecutorsUI

In particular, EnvironmentUI was not rendering until a job begins, and ExecutorsUI reports an incorrect number (format) of total tasks.
apache · Feb 15, 2014 · 8a2ebe6 · 8a2ebe6
1 parent de8a1cd
commit 8a2ebe6
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 24 deletions.
diff --git a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentUI.scala b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentUI.scala
@@ -49,6 +49,7 @@ private[spark] class EnvironmentUI(sc: SparkContext) {
    * Render an HTML page that encodes environment information
    */
   def render(request: HttpServletRequest): Seq[Node] = {
+    listener.loadEnvironment()
     val runtimeInformationTable = UIUtils.listingTable(
       propertyHeader, jvmRow, listener.jvmInformation, fixedWidth = true)
     val sparkPropertiesTable = UIUtils.listingTable(
@@ -83,19 +84,10 @@ private[spark] class EnvironmentUI(sc: SparkContext) {
     var systemProperties: Seq[(String, String)] = Seq()
     var classpathEntries: Seq[(String, String)] = Seq()
 
-    def onLoadEnvironment(loadEnvironment: SparkListenerLoadEnvironment) = {
-      jvmInformation = loadEnvironment.jvmInformation
-      sparkProperties = loadEnvironment.sparkProperties
-      systemProperties = loadEnvironment.systemProperties
-      classpathEntries = loadEnvironment.classpathEntries
-      logEvent(loadEnvironment)
-      logger.flush()
-    }
-
-    override def onJobStart(jobStart: SparkListenerJobStart) = {
-      logger.start()
-
-      // Gather properties
+    /**
+     * Gather JVM, spark, system and classpath properties
+     */
+    def loadEnvironment() = {
       val jvmInformation = Seq(
         ("Java Version", "%s (%s)".format(Properties.javaVersion, Properties.javaVendor)),
         ("Java Home", Properties.javaHome),
@@ -124,6 +116,20 @@ private[spark] class EnvironmentUI(sc: SparkContext) {
       onLoadEnvironment(loadEnvironment)
     }
 
+    /**
+     * Prepare environment information for UI to render, and log the corresponding event
+     */
+    def onLoadEnvironment(loadEnvironment: SparkListenerLoadEnvironment) = {
+      jvmInformation = loadEnvironment.jvmInformation
+      sparkProperties = loadEnvironment.sparkProperties
+      systemProperties = loadEnvironment.systemProperties
+      classpathEntries = loadEnvironment.classpathEntries
+      logEvent(loadEnvironment)
+      logger.flush()
+    }
+
+    override def onJobStart(jobStart: SparkListenerJobStart) = logger.start()
+
     override def onJobEnd(jobEnd: SparkListenerJobEnd) = logger.close()
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala
@@ -24,18 +24,17 @@ import scala.xml.Node
 
 import org.eclipse.jetty.server.Handler
 
-import org.apache.spark.SparkContext
+import org.apache.spark.{Logging, SparkContext, ExceptionFailure}
 import org.apache.spark.scheduler._
 import org.apache.spark.ui.JettyUtils._
 import org.apache.spark.ui.Page.Executors
 import org.apache.spark.ui.{UISparkListener, UIUtils}
 import org.apache.spark.util.Utils
 import org.apache.spark.scheduler.SparkListenerTaskEnd
-import org.apache.spark.ExceptionFailure
 import org.apache.spark.scheduler.SparkListenerTaskStart
 import org.apache.spark.storage.StorageStatus
 
-private[spark] class ExecutorsUI(val sc: SparkContext) {
+private[spark] class ExecutorsUI(val sc: SparkContext) extends Logging {
 
   private var _listener: Option[ExecutorsListener] = None
   def listener = _listener.get
@@ -105,10 +104,6 @@ private[spark] class ExecutorsUI(val sc: SparkContext) {
     val maximumMemory = values("Maximum Memory")
     val memoryUsed = values("Memory Used")
     val diskUsed = values("Disk Used")
-    val activeTasks = values("Active Tasks")
-    val failedTasks = values("Failed Tasks")
-    val completeTasks = values("Complete Tasks")
-    val totalTasks = activeTasks + failedTasks + completeTasks
 
     <tr>
       <td>{values("Executor ID")}</td>
@@ -121,10 +116,10 @@ private[spark] class ExecutorsUI(val sc: SparkContext) {
       <td sorttable_customkey={diskUsed}>
         {Utils.bytesToString(diskUsed.toLong)}
       </td>
-      <td>{activeTasks}</td>
-      <td>{failedTasks}</td>
-      <td>{completeTasks}</td>
-      <td>{totalTasks}</td>
+      <td>{values("Active Tasks")}</td>
+      <td>{values("Failed Tasks")}</td>
+      <td>{values("Complete Tasks")}</td>
+      <td>{values("Total Tasks")}</td>
       <td>{Utils.msDurationToString(values("Task Time").toLong)}</td>
       <td>{Utils.bytesToString(values("Shuffle Read").toLong)}</td>
       <td>{Utils.bytesToString(values("Shuffle Write").toLong)}</td>