Improved scalability

apache · Apr 21, 2015 · 999ccd4 · 999ccd4
1 parent dec85db
commit 999ccd4
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 4 deletions.
diff --git a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
@@ -57,7 +57,7 @@ function drawJobTimeline(groupArray, eventObjArray) {
   setupZoomable("#job-timeline-zoom-lock", jobTimeline);
 }
 
-function drawTaskAssignmentTimeline(groupArray, eventObjArray) {
+function drawTaskAssignmentTimeline(groupArray, eventObjArray, minLaunchTime, zoomMax) {
   var groups = new vis.DataSet(groupArray);
   var items = new vis.DataSet(eventObjArray);
   var container = $("#task-assignment-timeline")[0]
@@ -69,14 +69,19 @@ function drawTaskAssignmentTimeline(groupArray, eventObjArray) {
     align: 'left',
     selectable: false,
     showCurrentTime: false,
-    zoomable: false
+    zoomable: false,
+    zoomMax: zoomMax
   };
 
   var taskTimeline = new vis.Timeline(container)
   taskTimeline.setOptions(options);
   taskTimeline.setGroups(groups);
   taskTimeline.setItems(items);
-
+  var curEnd = taskTimeline.getWindow()["end"].getTime();
+  if (curEnd - minLaunchTime > zoomMax) {
+    curEnd = minLaunchTime + zoomMax;
+  }
+  taskTimeline.setWindow(minLaunchTime, curEnd);
   setupZoomable('#task-assignment-timeline-zoom-lock', taskTimeline);
 }
 

diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -436,6 +436,9 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
 
       val executorsSet = new HashSet[(String, String)]
 
+      var minLaunchTime = Long.MaxValue
+      var maxFinishTime = Long.MinValue
+      var numEffectiveTasks = 0
       val executorsArrayStr = stageData.taskData.flatMap {
         case (_, taskUIData) =>
           val taskInfo = taskUIData.taskInfo
@@ -464,6 +467,9 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
             val launchTime = taskInfo.launchTime
             val finishTime = if (!isRunning) taskInfo.finishTime else currentTime
             val totalExecutionTime = finishTime - launchTime
+            minLaunchTime = launchTime.min(minLaunchTime)
+            maxFinishTime = launchTime.max(maxFinishTime)
+            numEffectiveTasks += 1
 
             val metricsOpt = taskUIData.taskMetrics
             val shuffleReadTime =
@@ -555,6 +561,20 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
           """.stripMargin
       }.mkString("[", ",", "]")
 
+      var maxWindowInSec = ((maxFinishTime - minLaunchTime) / 1000.0).round
+      if (maxWindowInSec <= 0) maxWindowInSec = 1
+      val tasksPerSecond = numEffectiveTasks / maxWindowInSec
+      var maxZoom = {
+        if (tasksPerSecond > 100) {
+          1000L / (tasksPerSecond / 100)
+        }
+        else {
+          24L * 60 * 60 * 1000
+        }
+      }
+
+      if (maxZoom < 0) maxZoom = 1
+
       val content =
         summary ++
         showAdditionalMetrics ++
@@ -570,7 +590,8 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
           </div>
         </div> ++
         <script type="text/javascript">
-          {Unparsed(s"drawTaskAssignmentTimeline(${groupArrayStr}, ${executorsArrayStr})")}
+          {Unparsed(s"drawTaskAssignmentTimeline(" +
+            s"${groupArrayStr}, ${executorsArrayStr}, ${minLaunchTime}, ${maxZoom})")}
         </script>
 
       UIUtils.headerSparkPage("Details for Stage %d".format(stageId), content, parent)