From fdfc18163ed9efd68a5a3e6d83d4acc4b2013bf7 Mon Sep 17 00:00:00 2001 From: Imran Rashid Date: Wed, 1 Apr 2015 13:50:22 -0500 Subject: [PATCH] stage/taskList --- .../status/api/v1/OneStageResource.scala | 94 +- .../stages/20/0/taskList/json_expectation | 481 +++++++ .../json_expectation | 1201 +++++++++++++++++ .../taskList?sortBy=-runtime/json_expectation | 481 +++++++ .../json_expectation | 481 +++++++ .../taskList?sortBy=runtime/json_expectation | 481 +++++++ .../deploy/history/HistoryServerSuite.scala | 12 + 7 files changed, 3215 insertions(+), 16 deletions(-) create mode 100644 core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList/json_expectation create mode 100644 core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?offset=10&length=50/json_expectation create mode 100644 core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?sortBy=-runtime/json_expectation create mode 100644 core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?sortBy=DecreasingRuntime/json_expectation create mode 100644 core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?sortBy=runtime/json_expectation diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/OneStageResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/OneStageResource.scala index 222b284848649..1a84795007431 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/OneStageResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/OneStageResource.scala @@ -22,6 +22,8 @@ import javax.ws.rs.core.MediaType import org.apache.spark.SparkException import org.apache.spark.scheduler.StageInfo import org.apache.spark.ui.jobs.JobProgressListener +import org.apache.spark.ui.jobs.UIData.StageUIData +import org.apache.spark.util.SparkEnum @Produces(Array(MediaType.APPLICATION_JSON)) private[v1] class OneStageResource(uiRoot: UIRoot) { @@ -53,13 +55,7 @@ private[v1] class OneStageResource(uiRoot: UIRoot) { @PathParam("stageId") stageId: Int, @PathParam("attemptId") attemptId: Int ): StageData = { - forStageAttempt(appId, stageId, attemptId) { case (listener, status, stageInfo) => - val stageUiData = listener.synchronized { - listener.stageIdToData.get((stageInfo.stageId, stageInfo.attemptId)). - getOrElse(throw new SparkException("failed to get full stage data for stage: " + - stageInfo.stageId + ":" + stageInfo.attemptId) - ) - } + forStageAttempt(appId, stageId, attemptId) { case (status, stageInfo, stageUiData) => AllStagesResource.stageUiToStageData(status, stageInfo, stageUiData, includeDetails = true) } @@ -73,13 +69,7 @@ private[v1] class OneStageResource(uiRoot: UIRoot) { @PathParam("attemptId") attemptId: Int, @DefaultValue("0.05,0.25,0.5,0.75,0.95") @QueryParam("quantiles") quantileString: String ): TaskMetricDistributions = { - forStageAttempt(appId, stageId, attemptId) { case (listener, status, stageInfo) => - val stageUiData = listener.synchronized { - listener.stageIdToData.get((stageInfo.stageId, stageInfo.attemptId)). - getOrElse(throw new SparkException("failed to get full stage data for stage: " + - stageInfo.stageId + ":" + stageInfo.attemptId) - ) - } + forStageAttempt(appId, stageId, attemptId) { case (status, stageInfo, stageUiData) => val quantiles = quantileString.split(",").map{s => try { s.toDouble @@ -92,6 +82,24 @@ private[v1] class OneStageResource(uiRoot: UIRoot) { } } + @GET + @Path("/{attemptId: \\d+}/taskList") + def taskList( + @PathParam("appId") appId: String, + @PathParam("stageId") stageId: Int, + @PathParam("attemptId") attemptId: Int, + @DefaultValue("0") @QueryParam("offset") offset: Int, + @DefaultValue("20") @QueryParam("length") length: Int, + @DefaultValue("ID") @QueryParam("sortBy") sortBy: TaskSorting + ): Seq[TaskData] = { + forStageAttempt(appId, stageId, attemptId) { case (status, stageInfo, stageUiData) => + val tasks = stageUiData.taskData.values.map{AllStagesResource.convertTaskData}.toIndexedSeq + .sorted(sortBy.ordering) + tasks.slice(offset, offset + length) + } + } + + def forStage[T](appId: String, stageId: Int) (f: (JobProgressListener, Seq[(StageStatus, StageInfo)]) => T): T = { uiRoot.withSparkUI(appId) { ui => @@ -111,14 +119,20 @@ private[v1] class OneStageResource(uiRoot: UIRoot) { } def forStageAttempt[T](appId: String, stageId: Int, attemptId: Int) - (f: (JobProgressListener, StageStatus, StageInfo) => T): T = { + (f: (StageStatus, StageInfo, StageUIData) => T): T = { forStage(appId, stageId) { case (listener, attempts) => val oneAttempt = attempts.filter{ case (status, stage) => stage.attemptId == attemptId }.headOption oneAttempt match { case Some((status, stageInfo)) => - f(listener, status, stageInfo) + val stageUiData = listener.synchronized { + listener.stageIdToData.get((stageInfo.stageId, stageInfo.attemptId)). + getOrElse(throw new SparkException("failed to get full stage data for stage: " + + stageInfo.stageId + ":" + stageInfo.attemptId) + ) + } + f(status, stageInfo, stageUiData) case None => val stageAttempts = attempts.map { _._2.attemptId} throw new NotFoundException(s"unknown attempt for stage $stageId. " + @@ -127,3 +141,51 @@ private[v1] class OneStageResource(uiRoot: UIRoot) { } } } + +sealed abstract class TaskSorting extends SparkEnum { + def ordering: Ordering[TaskData] + def alternateNames: Seq[String] = Seq() +} +object TaskSorting extends JerseyEnum[TaskSorting] { + final val ID = { + case object ID extends TaskSorting { + def ordering = Ordering.by{td: TaskData => td.taskId} + } + ID + } + + final val IncreasingRuntime = { + case object IncreasingRuntime extends TaskSorting { + def ordering = Ordering.by{td: TaskData => + td.taskMetrics.map{_.executorRunTime}.getOrElse(-1L) + } + override def alternateNames = Seq("runtime", "+runtime") + } + IncreasingRuntime + } + + final val DecreasingRuntime = { + case object DecreasingRuntime extends TaskSorting { + def ordering = IncreasingRuntime.ordering.reverse + override def alternateNames = Seq("-runtime") + } + DecreasingRuntime + } + + val values = Seq( + ID, + IncreasingRuntime, + DecreasingRuntime + ) + + val alternateNames: Map[String, TaskSorting] = values.flatMap{x => x.alternateNames.map{_ -> x}}.toMap + + override def fromString(s: String): TaskSorting = { + alternateNames.find { case (k, v) => + k.toLowerCase() == s.toLowerCase() + }.map{_._2}.getOrElse{ + super.fromString(s) + } + } +} + diff --git a/core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList/json_expectation new file mode 100644 index 0000000000000..140c76d7e5208 --- /dev/null +++ b/core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList/json_expectation @@ -0,0 +1,481 @@ +[ { + "taskId" : 5010, + "index" : 0, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.571GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 7, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 318464, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5011, + "index" : 1, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.571GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 271674, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5012, + "index" : 2, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.571GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 7, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 274899, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5013, + "index" : 3, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.571GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 11, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9597, + "writeTime" : 315224, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5014, + "index" : 4, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.571GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 12, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9599, + "writeTime" : 316561, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5015, + "index" : 5, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.571GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 7, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 269443, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5016, + "index" : 6, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.571GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 275977, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5017, + "index" : 7, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.571GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 267412, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5018, + "index" : 8, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.579GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 310279, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5019, + "index" : 9, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.579GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 319028, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5020, + "index" : 10, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.579GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 306724, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5021, + "index" : 11, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.579GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9599, + "writeTime" : 280729, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5022, + "index" : 12, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.579GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 6, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 273837, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5023, + "index" : 13, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.579GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 6, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 282108, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5024, + "index" : 14, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.586GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9604, + "writeTime" : 266308, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5025, + "index" : 15, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.586GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 8, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 330830, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5026, + "index" : 16, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.586GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 247775, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5027, + "index" : 17, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.586GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 289503, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5028, + "index" : 18, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.592GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 355545, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5029, + "index" : 19, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.593GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 7, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 339081, + "recordsWritten" : 100 + } + } +} ] \ No newline at end of file diff --git a/core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?offset=10&length=50/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?offset=10&length=50/json_expectation new file mode 100644 index 0000000000000..0fa1ac28c9e90 --- /dev/null +++ b/core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?offset=10&length=50/json_expectation @@ -0,0 +1,1201 @@ +[ { + "taskId" : 5020, + "index" : 10, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.579GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 306724, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5021, + "index" : 11, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.579GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9599, + "writeTime" : 280729, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5022, + "index" : 12, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.579GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 6, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 273837, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5023, + "index" : 13, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.579GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 6, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 282108, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5024, + "index" : 14, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.586GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9604, + "writeTime" : 266308, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5025, + "index" : 15, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.586GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 8, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 330830, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5026, + "index" : 16, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.586GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 247775, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5027, + "index" : 17, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.586GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 289503, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5028, + "index" : 18, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.592GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 355545, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5029, + "index" : 19, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.593GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 7, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 339081, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5030, + "index" : 20, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.593GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 9, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 367295, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5031, + "index" : 21, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.593GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 9, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9597, + "writeTime" : 493701, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5032, + "index" : 22, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.593GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 345941, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5033, + "index" : 23, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.593GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 301382, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5034, + "index" : 24, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.594GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9603, + "writeTime" : 340547, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5035, + "index" : 25, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.600GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 324374, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5036, + "index" : 26, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.600GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9603, + "writeTime" : 321083, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5037, + "index" : 27, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.601GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 7, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 318662, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5038, + "index" : 28, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.601GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9599, + "writeTime" : 334376, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5039, + "index" : 29, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.601GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 7, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 287042, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5040, + "index" : 30, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.601GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 332938, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5041, + "index" : 31, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.602GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 4, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9603, + "writeTime" : 255099, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5042, + "index" : 32, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.602GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 297534, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5043, + "index" : 33, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.607GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 265929, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5044, + "index" : 34, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.607GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 279851, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5045, + "index" : 35, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.608GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 278965, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5046, + "index" : 36, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.609GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9599, + "writeTime" : 340518, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5047, + "index" : 37, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.609GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9597, + "writeTime" : 402877, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5048, + "index" : 38, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.609GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 7, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 265447, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5049, + "index" : 39, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.611GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 3, + "executorRunTime" : 23, + "resultSize" : 930, + "jvmGcTime" : 16, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 304745, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5050, + "index" : 40, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.615GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 24, + "resultSize" : 930, + "jvmGcTime" : 16, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 302371, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5051, + "index" : 41, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.615GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 25, + "resultSize" : 930, + "jvmGcTime" : 16, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 296177, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5052, + "index" : 42, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.615GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 24, + "resultSize" : 930, + "jvmGcTime" : 16, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 962033, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5053, + "index" : 43, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.616GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 26, + "resultSize" : 930, + "jvmGcTime" : 16, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 310335, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5054, + "index" : 44, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.616GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 110, + "resultSize" : 930, + "jvmGcTime" : 22, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 299315, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5055, + "index" : 45, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.617GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 25, + "resultSize" : 930, + "jvmGcTime" : 16, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9603, + "writeTime" : 307821, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5056, + "index" : 46, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.617GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 17, + "executorRunTime" : 7, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9604, + "writeTime" : 306911, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5057, + "index" : 47, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.639GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 274191, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5058, + "index" : 48, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.640GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 48, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 10442141, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5059, + "index" : 49, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.641GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 7, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 300675, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5060, + "index" : 50, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.642GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 24, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 812196, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5061, + "index" : 51, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.642GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 6, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 313090, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5062, + "index" : 52, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.642GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 17, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 1572828, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5063, + "index" : 53, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.643GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 7, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9597, + "writeTime" : 312481, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5064, + "index" : 54, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.646GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 13, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 296976, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5065, + "index" : 55, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.650GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 7, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 670271, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5066, + "index" : 56, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.650GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 10, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 1240309, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5067, + "index" : 57, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.650GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 9, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 678632, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5068, + "index" : 58, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.659GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 7, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 302888, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5069, + "index" : 59, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.660GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 7, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9610, + "writeTime" : 383153, + "recordsWritten" : 100 + } + } +} ] \ No newline at end of file diff --git a/core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?sortBy=-runtime/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?sortBy=-runtime/json_expectation new file mode 100644 index 0000000000000..63b131621c9c4 --- /dev/null +++ b/core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?sortBy=-runtime/json_expectation @@ -0,0 +1,481 @@ +[ { + "taskId" : 5186, + "index" : 176, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.880GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 147, + "resultSize" : 930, + "jvmGcTime" : 12, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9599, + "writeTime" : 317390, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5463, + "index" : 453, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.518GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 134, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 348853, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5461, + "index" : 451, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.516GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 134, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9599, + "writeTime" : 413676, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5505, + "index" : 495, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.616GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 133, + "resultSize" : 930, + "jvmGcTime" : 11, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 384212, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5672, + "index" : 662, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:01.016GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 131, + "resultSize" : 930, + "jvmGcTime" : 4, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 387955, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5865, + "index" : 855, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:01.514GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 131, + "resultSize" : 930, + "jvmGcTime" : 10, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9599, + "writeTime" : 1710924, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5415, + "index" : 405, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.389GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 130, + "resultSize" : 930, + "jvmGcTime" : 14, + "resultSerializationTime" : 1, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9597, + "writeTime" : 377101, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5462, + "index" : 452, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.518GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 128, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 306075, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5411, + "index" : 401, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.385GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 128, + "resultSize" : 930, + "jvmGcTime" : 14, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 74684537, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5416, + "index" : 406, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.392GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 11, + "executorRunTime" : 123, + "resultSize" : 930, + "jvmGcTime" : 14, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 365240, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5924, + "index" : 914, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:01.648GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 4, + "executorRunTime" : 121, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 335682, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5179, + "index" : 169, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.866GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 114, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 276876, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5243, + "index" : 233, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.018GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 113, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 12003053, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5054, + "index" : 44, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.616GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 110, + "resultSize" : 930, + "jvmGcTime" : 22, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 299315, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5465, + "index" : 455, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.521GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 108, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 318366, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5614, + "index" : 604, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.893GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 19, + "executorRunTime" : 108, + "resultSize" : 930, + "jvmGcTime" : 7, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 309135, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5862, + "index" : 852, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:01.509GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 96, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 1052228, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5566, + "index" : 556, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.762GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 95, + "resultSize" : 930, + "jvmGcTime" : 6, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 327089, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5567, + "index" : 557, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.768GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 94, + "resultSize" : 930, + "jvmGcTime" : 6, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 374043, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5191, + "index" : 181, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.885GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 90, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 344199, + "recordsWritten" : 100 + } + } +} ] \ No newline at end of file diff --git a/core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?sortBy=DecreasingRuntime/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?sortBy=DecreasingRuntime/json_expectation new file mode 100644 index 0000000000000..63b131621c9c4 --- /dev/null +++ b/core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?sortBy=DecreasingRuntime/json_expectation @@ -0,0 +1,481 @@ +[ { + "taskId" : 5186, + "index" : 176, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.880GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 147, + "resultSize" : 930, + "jvmGcTime" : 12, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9599, + "writeTime" : 317390, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5463, + "index" : 453, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.518GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 134, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 348853, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5461, + "index" : 451, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.516GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 134, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9599, + "writeTime" : 413676, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5505, + "index" : 495, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.616GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 133, + "resultSize" : 930, + "jvmGcTime" : 11, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 384212, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5672, + "index" : 662, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:01.016GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 131, + "resultSize" : 930, + "jvmGcTime" : 4, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 387955, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5865, + "index" : 855, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:01.514GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 131, + "resultSize" : 930, + "jvmGcTime" : 10, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9599, + "writeTime" : 1710924, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5415, + "index" : 405, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.389GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 130, + "resultSize" : 930, + "jvmGcTime" : 14, + "resultSerializationTime" : 1, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9597, + "writeTime" : 377101, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5462, + "index" : 452, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.518GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 128, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 306075, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5411, + "index" : 401, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.385GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 128, + "resultSize" : 930, + "jvmGcTime" : 14, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 74684537, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5416, + "index" : 406, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.392GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 11, + "executorRunTime" : 123, + "resultSize" : 930, + "jvmGcTime" : 14, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 365240, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5924, + "index" : 914, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:01.648GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 4, + "executorRunTime" : 121, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 335682, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5179, + "index" : 169, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.866GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 114, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 276876, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5243, + "index" : 233, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.018GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 113, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 12003053, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5054, + "index" : 44, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.616GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 110, + "resultSize" : 930, + "jvmGcTime" : 22, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 299315, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5465, + "index" : 455, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.521GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 108, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 318366, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5614, + "index" : 604, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.893GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 19, + "executorRunTime" : 108, + "resultSize" : 930, + "jvmGcTime" : 7, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 309135, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5862, + "index" : 852, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:01.509GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 96, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 1052228, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5566, + "index" : 556, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.762GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 95, + "resultSize" : 930, + "jvmGcTime" : 6, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 327089, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5567, + "index" : 557, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.768GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 94, + "resultSize" : 930, + "jvmGcTime" : 6, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 374043, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5191, + "index" : 181, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.885GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 90, + "resultSize" : 930, + "jvmGcTime" : 5, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 344199, + "recordsWritten" : 100 + } + } +} ] \ No newline at end of file diff --git a/core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?sortBy=runtime/json_expectation b/core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?sortBy=runtime/json_expectation new file mode 100644 index 0000000000000..329065b835dd0 --- /dev/null +++ b/core/src/test/resources/HistoryServerExpectations/applications/local-1427397477963/stages/20/0/taskList?sortBy=runtime/json_expectation @@ -0,0 +1,481 @@ +[ { + "taskId" : 5610, + "index" : 600, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.884GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 285253, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5511, + "index" : 501, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.634GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9599, + "writeTime" : 277358, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5027, + "index" : 17, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.586GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 289503, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5828, + "index" : 818, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:01.438GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 3, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 291997, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5173, + "index" : 163, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.861GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9599, + "writeTime" : 267617, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5185, + "index" : 175, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.880GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 285928, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5553, + "index" : 543, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.719GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 298472, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5077, + "index" : 67, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.670GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9614, + "writeTime" : 259332, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5831, + "index" : 821, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:01.442GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9597, + "writeTime" : 281603, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5194, + "index" : 184, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.903GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9600, + "writeTime" : 286315, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5481, + "index" : 471, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.561GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9613, + "writeTime" : 328893, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5035, + "index" : 25, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.600GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 324374, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5134, + "index" : 124, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.782GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9599, + "writeTime" : 279741, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5044, + "index" : 34, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.607GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 279851, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5026, + "index" : 16, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.586GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9598, + "writeTime" : 247775, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5331, + "index" : 321, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.208GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 313699, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5343, + "index" : 333, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.241GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 0, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9601, + "writeTime" : 260259, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5606, + "index" : 596, + "attempt" : 0, + "launchTime" : "2015-03-26T19:20:00.878GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 1, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9599, + "writeTime" : 310521, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5041, + "index" : 31, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.602GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 4, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9603, + "writeTime" : 255099, + "recordsWritten" : 100 + } + } +}, { + "taskId" : 5023, + "index" : 13, + "attempt" : 0, + "launchTime" : "2015-03-26T19:19:59.579GMT", + "executorId" : "", + "host" : "localhost", + "taskLocality" : "PROCESS_LOCAL", + "speculative" : false, + "accumulatorUpdates" : [ ], + "taskMetrics" : { + "executorDeserializeTime" : 6, + "executorRunTime" : 5, + "resultSize" : 930, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "shuffleWriteMetrics" : { + "bytesWritten" : 9602, + "writeTime" : 282108, + "recordsWritten" : 100 + } + } +} ] \ No newline at end of file diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala index 696ed483b6684..fe5f36e6ed5c6 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala @@ -80,9 +80,21 @@ class HistoryServerSuite extends FunSuite with BeforeAndAfter with Matchers with "failed stage list json" -> "applications/local-1422981780767/stages?status=failed", "one stage json" -> "applications/local-1422981780767/stages/1", "one stage attempt json" -> "applications/local-1422981780767/stages/1/0", + "stage task summary" -> "applications/local-1427397477963/stages/20/0/taskSummary", "stage task summary w/ custom quantiles" -> "applications/local-1427397477963/stages/20/0/taskSummary?quantiles=0.01,0.5,0.99", + + "stage task list" -> "applications/local-1427397477963/stages/20/0/taskList", + "stage task list w/ offset & length" -> + "applications/local-1427397477963/stages/20/0/taskList?offset=10&length=50", + "stage task list w/ sortBy" -> + "applications/local-1427397477963/stages/20/0/taskList?sortBy=DecreasingRuntime", + "stage task list w/ sortBy short names: -runtime" -> + "applications/local-1427397477963/stages/20/0/taskList?sortBy=-runtime", + "stage task list w/ sortBy short names: runtime" -> + "applications/local-1427397477963/stages/20/0/taskList?sortBy=runtime", + "stage list with accumulable json" -> "applications/local-1426533911241/stages", "stage with accumulable json" -> "applications/local-1426533911241/stages/0/0", "rdd list storage json" -> "applications/local-1422981780767/storage/rdd",