Skip to content

Commit

Permalink
Cleanuped
Browse files Browse the repository at this point in the history
  • Loading branch information
@sarutak
sarutak committed Apr 3, 2015
1 parent d05f2c2 commit 9fb522e
Show file tree
Hide file tree
Showing 4 changed files with 238 additions and 265 deletions.
10 changes: 8 additions & 2 deletions core/src/main/resources/org/apache/spark/ui/static/timeline-view.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,10 +139,16 @@ div#application-timeline, div#job-timeline {
z-index: auto;
}

.executor.application-timeline-object.added {
.vis.timeline .item.executor.added {
background-color: #D5DDF6;
}

.executor.application-timeline-object.removed {
.vis.timeline .item.executor.removed {
background-color: #EBCA59;
}

.vis.timeline .item.executor.selected {
border-color: #FFC200;
background-color: #FFF785;
z-index: 2;
}
66 changes: 0 additions & 66 deletions core/src/main/scala/org/apache/spark/ui/TimelineViewUtils.scala
View file

This file was deleted.

223 changes: 119 additions & 104 deletions core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import java.util.Date
import javax.servlet.http.HttpServletRequest

import org.apache.spark.ui.{UIUtils, WebUIPage}
import org.apache.spark.ui.TimelineViewUtils._
import org.apache.spark.ui.jobs.UIData.JobUIData
import org.apache.spark.JobExecutionStatus

Expand All @@ -32,6 +31,123 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
private val startTime: Option[Long] = parent.sc.map(_.startTime)
private val listener = parent.listener

private def applicationTimelineView(jobs: Seq[JobUIData], now: Long): Seq[Node] = {
val jobEventJsonAsStrSeq = jobs.flatMap { jobUIData =>
val jobId = jobUIData.jobId
val status = jobUIData.status
val submissionTimeOpt = jobUIData.submissionTime
val completionTimeOpt = jobUIData.completionTime

if (status == JobExecutionStatus.UNKNOWN || submissionTimeOpt.isEmpty ||
completionTimeOpt.isEmpty && status != JobExecutionStatus.RUNNING) {
None
}

val submissionTime = submissionTimeOpt.get
val completionTime = completionTimeOpt.getOrElse(now)
val classNameByStatus = status match {
case JobExecutionStatus.SUCCEEDED => "succeeded"
case JobExecutionStatus.FAILED => "failed"
case JobExecutionStatus.RUNNING => "running"
}

val jobEventJsonAsStr =
s"""
|{
| 'className': 'job application-timeline-object ${classNameByStatus}',
| 'group': 'jobs',
| 'start': new Date(${submissionTime}),
| 'end': new Date(${completionTime}),
| 'content': '<div class="application-timeline-content">Job ${jobId}</div>',
| 'title': 'Job ${jobId}\\nStatus: ${status}\\n' +
| 'Submission Time: ${UIUtils.formatDate(new Date(submissionTime))}' +
| '${
if (status != JobExecutionStatus.RUNNING) {
s"""\\nCompletion Time: ${UIUtils.formatDate(new Date(completionTime))}"""
} else {
""
}
}'
|}
""".stripMargin
Option(jobEventJsonAsStr)
}

val executorEventJsonAsStrSeq =
(listener.executorIdToAddedTime ++
listener.executorIdToRemovedTimeAndReason).map { event =>
val (executorId: String, status: String, time: Long, reason: Option[String]) =
event match {
case (executorId, (removedTime, reason)) =>
(executorId, "removed", removedTime, Some(reason))
case (executorId, addedTime) =>
(executorId, "added", addedTime, None)
}
s"""
|{
| 'className': 'executor ${status}',
| 'group': 'executors',
| 'start': new Date(${time}),
| 'content': '<div>Executor ${executorId} ${status}</div>',
| 'title': '${if (status == "added") "Added" else "Removed"} ' +
| 'at ${UIUtils.formatDate(new Date(time))}' +
| '${if (reason.isDefined) s"""\\nReason: ${reason.get}""" else ""}'
|}
""".stripMargin
}

val executorsLegend =
<div class="legend-area"><svg width="200px" height="55px">
<rect x="5px" y="5px" width="20px" height="15px"
rx="2px" ry="2px" stroke="#97B0F8" fill="#D5DDF6"></rect>
<text x="35px" y="17px">Executor Added</text>
<rect x="5px" y="35px" width="20px" height="15px"
rx="2px" ry="2px" stroke="#97B0F8" fill="#EBCA59"></rect>
<text x="35px" y="47px">Executor Removed</text>
</svg></div>.toString.filter(_ != '\n')

val jobsLegend =
<div class="legend-area"><svg width="200px" height="85px">
<rect x="5px" y="5px" width="20px" height="15px"
rx="2px" ry="2px" stroke="#97B0F8" fill="#D5DDF6"></rect>
<text x="35px" y="17px">Succeeded Job</text>
<rect x="5px" y="35px" width="20px" height="15px"
rx="2px" ry="2px" stroke="#97B0F8" fill="#FF5475"></rect>
<text x="35px" y="47px">Failed Job</text>
<rect x="5px" y="65px" width="20px" height="15px"
rx="2px" ry="2px" stroke="#97B0F8" fill="#FDFFCA"></rect>
<text x="35px" y="77px">Running Job</text>
</svg></div>.toString.filter(_ != '\n')

val groupJsonArrayAsStr =
s"""
|[
| {
| 'id': 'executors',
| 'content': '<div>Executors</div>${executorsLegend}',
| },
| {
| 'id': 'jobs',
| 'content': '<div>Jobs</div>${jobsLegend}',
| }
|]
""".stripMargin

val eventArrayAsStr =
(jobEventJsonAsStrSeq ++ executorEventJsonAsStrSeq).mkString("[", ",", "]")

<div class="control-panel">
<div id="application-timeline-zoom-lock">
<input type="checkbox" checked="checked"></input>
<span>Zoom Lock</span>
</div>
</div> ++
<div id="application-timeline"></div> ++
<script type="text/javascript">
{Unparsed(s"drawApplicationTimeline(${groupJsonArrayAsStr}, ${eventArrayAsStr});")}
</script>
}

private def jobsTable(jobs: Seq[JobUIData]): Seq[Node] = {
val someJobHasJobGroup = jobs.exists(_.jobGroup.isDefined)

Expand Down Expand Up @@ -97,15 +213,6 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
</table>
}

private val controlPanel : Seq[Node] = {
<div class="control-panel">
<div id="application-timeline-zoom-lock">
<input type="checkbox" checked="checked"></input>
<span>Zoom Lock</span>
</div>
</div>
}

def render(request: HttpServletRequest): Seq[Node] = {
listener.synchronized {
val activeJobs = listener.activeJobs.values.toSeq
Expand Down Expand Up @@ -166,100 +273,8 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
</div>

var content = summary

val groupArrayStr =
s"""
|[
| {
| 'id': 'executors',
| 'content': '<div>Executors</div>${nodesToFlatString(executorsLegend)}',
| },
| {
| 'id': 'jobs',
| 'content': '<div>Jobs</div>${nodesToFlatString(jobsLegend)}',
| }
|]
""".stripMargin

val jobEventArray = (completedJobs ++ failedJobs ++ activeJobs).flatMap { jobUIData =>
val jobId = jobUIData.jobId
val status = jobUIData.status
val submissionTimeOpt = jobUIData.submissionTime
val completionTimeOpt = jobUIData.completionTime

if (status == JobExecutionStatus.UNKNOWN || submissionTimeOpt.isEmpty ||
completionTimeOpt.isEmpty && status != JobExecutionStatus.RUNNING) {
None
}

val submissionTime = submissionTimeOpt.get
val completionTime = completionTimeOpt.getOrElse(now)
val classNameByStatus = status match {
case JobExecutionStatus.SUCCEEDED => "succeeded"
case JobExecutionStatus.FAILED => "failed"
case JobExecutionStatus.RUNNING => "running"
}

val timelineObject =
s"""
|{
| 'className': 'job application-timeline-object ${classNameByStatus}',
| 'group': 'jobs',
| 'start': new Date(${submissionTime}),
| 'end': new Date(${completionTime}),
| 'content': '<div class="application-timeline-content">' +
| 'Job ${jobId}</div>',
| 'title': 'Job ${jobId}\\nStatus: ${status}\\n' +
| 'Submission Time: ${UIUtils.formatDate(new Date(submissionTime))}' +
| '${
if (status != JobExecutionStatus.RUNNING) {
s"""\\nCompletion Time: ${UIUtils.formatDate(new Date(completionTime))}"""
} else {
""
}
}'
|}
""".stripMargin
Option(timelineObject)
}

val executorAddedEventArray = listener.executorIdToAddedTime.map {
case (executorId, addedTime) =>
s"""
|{
| 'className': 'executor application-tmeline-object added',
| 'group': 'executors',
| 'start': new Date(${addedTime}),
| 'content': '<div>Executor ${executorId} added</div>',
| 'title': 'Added at ${UIUtils.formatDate(new Date(addedTime))}'
|}
""".stripMargin
}

val executorRemovedEventArray = listener.executorIdToRemovedTimeAndReason.map {
case (executorId, (removedTime, reason)) =>
s"""
|{
| 'className': 'executor application-timeline-object removed',
| 'group': 'executors',
| 'start': new Date(${removedTime}),
| 'content': '<div>Executor ${executorId} removed (${reason})</div>',
| 'title': 'Removed at ${UIUtils.formatDate(new Date(removedTime))}\\n' +
| 'Reason: ${reason}'
|}
""".stripMargin
}

val eventArrayStr =
(jobEventArray ++ executorAddedEventArray ++
executorRemovedEventArray).mkString("[", ",", "]")

content ++= <h4>Events on Application Timeline</h4> ++ controlPanel ++
<div id="application-timeline"></div>
content ++=
<script type="text/javascript">
{Unparsed(s"drawApplicationTimeline(${groupArrayStr}, ${eventArrayStr});")}
</script>
content ++= <h4>Events on Application Timeline</h4> ++
applicationTimelineView(activeJobs ++ completedJobs ++ failedJobs, now)

if (shouldShowActiveJobs) {
content ++= <h4 id="active">Active Jobs ({activeJobs.size})</h4> ++
Expand Down
Loading

0 comments on commit 9fb522e

Please sign in to comment.