From 55a1c31ee04caff9f6893eb7cdf12d147a930f7a Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Tue, 1 Dec 2015 17:19:43 -0800 Subject: [PATCH 01/14] SPARK-1832 added color to tasks on executor page --- .../org/apache/spark/status/api/v1/api.scala | 1 + .../apache/spark/ui/exec/ExecutorsPage.scala | 39 +++++++++++++++++-- .../apache/spark/ui/exec/ExecutorsTab.scala | 2 + 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index 5feb1dc2e5b74..f065b499655ea 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -55,6 +55,7 @@ class ExecutorSummary private[spark]( val rddBlocks: Int, val memoryUsed: Long, val diskUsed: Long, + val totalCores: Int, val activeTasks: Int, val failedTasks: Int, val completedTasks: Int, diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index 1a29b0f412603..e6325e0604df3 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -33,6 +33,7 @@ private[ui] case class ExecutorSummaryInfo( rddBlocks: Int, memoryUsed: Long, diskUsed: Long, + totalCores: Int, activeTasks: Int, failedTasks: Int, completedTasks: Int, @@ -117,6 +118,18 @@ private[ui] class ExecutorsPage( val maximumMemory = info.maxMemory val memoryUsed = info.memoryUsed val diskUsed = info.diskUsed + var activeTasksAlpha = 1.0 + var failedTasksAlpha = 1.0 + var completedTasksAlpha = 1.0 + if (info.totalCores > 0) { + activeTasksAlpha = 0.5 + 0.5 * info.activeTasks / info.totalCores + } + if (info.totalTasks > 0) { + failedTasksAlpha = math.min(0.1 * info.failedTasks / info.totalTasks, 1) * 0.5 + 0.5 + completedTasksAlpha + = math.max(10.0 * info.completedTasks / info.totalTasks - 9, 0) * 0.5 + 0.5 + } + {info.id} {info.hostPort} @@ -128,9 +141,27 @@ private[ui] class ExecutorsPage( {Utils.bytesToString(diskUsed)} - {info.activeTasks} - {info.failedTasks} - {info.completedTasks} + 0) { + "background:hsla(120, 100%, 25%, " + activeTasksAlpha + ");color:white" + } else { + "" + } + }>{info.activeTasks} + 0) { + "background:hsla(0, 100%, 50%, " + failedTasksAlpha + ");color:white" + } else { + "" + } + }>{info.failedTasks} + 0) { + "background:hsla(240, 100%, 50%, " + completedTasksAlpha + ");color:white" + } else { + "" + } + }>{info.completedTasks} {info.totalTasks} {Utils.msDurationToString(info.totalDuration)} @@ -184,6 +215,7 @@ private[spark] object ExecutorsPage { val memUsed = status.memUsed val maxMem = status.maxMem val diskUsed = status.diskUsed + val totalCores = listener.executorToTotalCores.getOrElse(execId, 0) val activeTasks = listener.executorToTasksActive.getOrElse(execId, 0) val failedTasks = listener.executorToTasksFailed.getOrElse(execId, 0) val completedTasks = listener.executorToTasksComplete.getOrElse(execId, 0) @@ -200,6 +232,7 @@ private[spark] object ExecutorsPage { rddBlocks, memUsed, diskUsed, + totalCores, activeTasks, failedTasks, completedTasks, diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala index a88fc4c37d3c9..7416f8be2e3f6 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala @@ -44,6 +44,7 @@ private[ui] class ExecutorsTab(parent: SparkUI) extends SparkUITab(parent, "exec */ @DeveloperApi class ExecutorsListener(storageStatusListener: StorageStatusListener) extends SparkListener { + val executorToTotalCores = HashMap[String, Int]() val executorToTasksActive = HashMap[String, Int]() val executorToTasksComplete = HashMap[String, Int]() val executorToTasksFailed = HashMap[String, Int]() @@ -62,6 +63,7 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener) extends Sp override def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit = synchronized { val eid = executorAdded.executorId executorToLogUrls(eid) = executorAdded.executorInfo.logUrlMap + executorToTotalCores(eid) = executorAdded.executorInfo.totalCores executorIdToData(eid) = ExecutorUIData(executorAdded.time) } From d9f2b82ee14e2909c309906e52cd3258bb359a8e Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Fri, 4 Dec 2015 12:42:16 -0800 Subject: [PATCH 02/14] SPARK-12149 Cleaned up color UI code, fixed tests, added color UI for GC time --- .../org/apache/spark/status/api/v1/api.scala | 1 + .../apache/spark/ui/exec/ExecutorsPage.scala | 34 +++++++++++++++---- .../apache/spark/ui/exec/ExecutorsTab.scala | 2 ++ .../executor_list_json_expectation.json | 2 ++ 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index f065b499655ea..43a915d5825cc 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -61,6 +61,7 @@ class ExecutorSummary private[spark]( val completedTasks: Int, val totalTasks: Int, val totalDuration: Long, + val totalGCTime: Long, val totalInputBytes: Long, val totalShuffleRead: Long, val totalShuffleWrite: Long, diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index e6325e0604df3..fc81d34782062 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -39,6 +39,7 @@ private[ui] case class ExecutorSummaryInfo( completedTasks: Int, totalTasks: Int, totalDuration: Long, + totalGCTime: Long, totalInputBytes: Long, totalShuffleRead: Long, totalShuffleWrite: Long, @@ -118,16 +119,26 @@ private[ui] class ExecutorsPage( val maximumMemory = info.maxMemory val memoryUsed = info.memoryUsed val diskUsed = info.diskUsed + + // Determine Color Opacity from 0.5-1 var activeTasksAlpha = 1.0 var failedTasksAlpha = 1.0 var completedTasksAlpha = 1.0 + var totalDurationAlpha = 1.0 if (info.totalCores > 0) { - activeTasksAlpha = 0.5 + 0.5 * info.activeTasks / info.totalCores + // activeTasks range from 0 to all cores + activeTasksAlpha = (info.activeTasks.toDouble / info.totalCores) * 0.5 + 0.5 } if (info.totalTasks > 0) { - failedTasksAlpha = math.min(0.1 * info.failedTasks / info.totalTasks, 1) * 0.5 + 0.5 + // failedTasks range max at 10% failure, alpha > 1 doesn't affect CSS + failedTasksAlpha = (10 * info.failedTasks.toDouble / info.totalTasks) * 0.5 + 0.5 + // completedTasks range ignores 90% of tasks completedTasksAlpha - = math.max(10.0 * info.completedTasks / info.totalTasks - 9, 0) * 0.5 + 0.5 + = math.max(((10 * info.completedTasks.toDouble / info.totalTasks) - 9) * 0.5 + 0.5, 0.5) + } + if (info.totalDuration > 0) { + // totalDuration range from 0 to 50% GC time, alpha > 1 doesn't affect CSS + totalDurationAlpha = info.totalGCTime.toDouble / info.totalDuration + 0.5 } @@ -141,21 +152,21 @@ private[ui] class ExecutorsPage( {Utils.bytesToString(diskUsed)} - 0) { "background:hsla(120, 100%, 25%, " + activeTasksAlpha + ");color:white" } else { "" } }>{info.activeTasks} - 0) { "background:hsla(0, 100%, 50%, " + failedTasksAlpha + ");color:white" } else { "" } }>{info.failedTasks} - 0) { "background:hsla(240, 100%, 50%, " + completedTasksAlpha + ");color:white" } else { @@ -163,7 +174,14 @@ private[ui] class ExecutorsPage( } }>{info.completedTasks} {info.totalTasks} - + info.totalDuration) { + "background:hsla(0, 100%, 50%, " + totalDurationAlpha + ");color:white" + } else { + "" + } + }> {Utils.msDurationToString(info.totalDuration)} @@ -221,6 +239,7 @@ private[spark] object ExecutorsPage { val completedTasks = listener.executorToTasksComplete.getOrElse(execId, 0) val totalTasks = activeTasks + failedTasks + completedTasks val totalDuration = listener.executorToDuration.getOrElse(execId, 0L) + val totalGCTime = listener.executorToJvmGCTime.getOrElse(execId, 0L) val totalInputBytes = listener.executorToInputBytes.getOrElse(execId, 0L) val totalShuffleRead = listener.executorToShuffleRead.getOrElse(execId, 0L) val totalShuffleWrite = listener.executorToShuffleWrite.getOrElse(execId, 0L) @@ -238,6 +257,7 @@ private[spark] object ExecutorsPage { completedTasks, totalTasks, totalDuration, + totalGCTime, totalInputBytes, totalShuffleRead, totalShuffleWrite, diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala index 7416f8be2e3f6..309fcec7b6a3f 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala @@ -49,6 +49,7 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener) extends Sp val executorToTasksComplete = HashMap[String, Int]() val executorToTasksFailed = HashMap[String, Int]() val executorToDuration = HashMap[String, Long]() + val executorToJvmGCTime = HashMap[String, Long]() val executorToInputBytes = HashMap[String, Long]() val executorToInputRecords = HashMap[String, Long]() val executorToOutputBytes = HashMap[String, Long]() @@ -133,6 +134,7 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener) extends Sp executorToShuffleWrite(eid) = executorToShuffleWrite.getOrElse(eid, 0L) + shuffleWrite.shuffleBytesWritten } + executorToJvmGCTime(eid) = executorToJvmGCTime.getOrElse(eid, 0L) + metrics.jvmGCTime } } } diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json index cb622e147249e..f407bfa951739 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json @@ -4,11 +4,13 @@ "rddBlocks" : 8, "memoryUsed" : 28000128, "diskUsed" : 0, + "totalCores" : 0, "activeTasks" : 0, "failedTasks" : 1, "completedTasks" : 31, "totalTasks" : 32, "totalDuration" : 8820, + "totalGCTime" : 352, "totalInputBytes" : 28000288, "totalShuffleRead" : 0, "totalShuffleWrite" : 13180, From 1da5f1a61a69a6835ccb3cacfda9a51823c544f4 Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Mon, 7 Dec 2015 12:39:20 -0800 Subject: [PATCH 03/14] Addressed comments --- .../apache/spark/ui/exec/ExecutorsPage.scala | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index fc81d34782062..1eba6bde4fde1 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -121,25 +121,29 @@ private[ui] class ExecutorsPage( val diskUsed = info.diskUsed // Determine Color Opacity from 0.5-1 - var activeTasksAlpha = 1.0 - var failedTasksAlpha = 1.0 - var completedTasksAlpha = 1.0 - var totalDurationAlpha = 1.0 - if (info.totalCores > 0) { - // activeTasks range from 0 to all cores - activeTasksAlpha = (info.activeTasks.toDouble / info.totalCores) * 0.5 + 0.5 - } - if (info.totalTasks > 0) { - // failedTasks range max at 10% failure, alpha > 1 doesn't affect CSS - failedTasksAlpha = (10 * info.failedTasks.toDouble / info.totalTasks) * 0.5 + 0.5 - // completedTasks range ignores 90% of tasks - completedTasksAlpha - = math.max(((10 * info.completedTasks.toDouble / info.totalTasks) - 9) * 0.5 + 0.5, 0.5) - } - if (info.totalDuration > 0) { - // totalDuration range from 0 to 50% GC time, alpha > 1 doesn't affect CSS - totalDurationAlpha = info.totalGCTime.toDouble / info.totalDuration + 0.5 - } + // activeTasks range from 0 to all cores + val activeTasksAlpha = + if (info.totalCores > 0) { + (info.activeTasks.toDouble / info.totalCores) * 0.5 + 0.5 + } else { + 1 + } + // failedTasks range max at 10% failure, alpha max = 1 + // completedTasks range ignores 90% of tasks + val (failedTasksAlpha, completedTasksAlpha) = + if (info.totalTasks > 0) { + (math.min(10 * info.failedTasks.toDouble / info.totalTasks, 1) * 0.5 + 0.5, + math.max(((10 * info.completedTasks.toDouble / info.totalTasks) - 9) * 0.5 + 0.5, 0.5)) + } else { + (1, 1) + } + // totalDuration range from 0 to 50% GC time, alpha max = 1 + val totalDurationAlpha = + if (info.totalDuration > 0) { + math.min(info.totalGCTime.toDouble / info.totalDuration + 0.5, 1) + } else { + 1 + } {info.id} From 20c7b48dcb29ce9e56be513b8cfb425012e482d5 Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Tue, 8 Dec 2015 14:20:30 -0800 Subject: [PATCH 04/14] added MiMa exclude --- project/MimaExcludes.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 685cb419ca8a7..8959ab1d8ffb3 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -161,6 +161,9 @@ object MimaExcludes { // SPARK-3580 Add getNumPartitions method to JavaRDD ProblemFilters.exclude[MissingMethodProblem]( "org.apache.spark.api.java.JavaRDDLike.getNumPartitions") + ) ++ Seq( + // SPARK-12149 Added new fields to ExecutorSummary + ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.status.api.v1.ExecutorSummary.this") ) ++ // SPARK-11314: YARN backend moved to yarn sub-module and MiMA complains even though it's a // private class. From 1652274abf3d335918d3b84e5b874aa4a58e9100 Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Thu, 10 Dec 2015 17:05:54 -0800 Subject: [PATCH 05/14] Comments followup --- .../src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index 1eba6bde4fde1..d9bdd2baf7eb6 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -33,13 +33,11 @@ private[ui] case class ExecutorSummaryInfo( rddBlocks: Int, memoryUsed: Long, diskUsed: Long, - totalCores: Int, activeTasks: Int, failedTasks: Int, completedTasks: Int, totalTasks: Int, totalDuration: Long, - totalGCTime: Long, totalInputBytes: Long, totalShuffleRead: Long, totalShuffleWrite: Long, From 190a03340e3795f21e3b8886b5f0c231136cba0b Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Mon, 21 Dec 2015 15:06:44 -0800 Subject: [PATCH 06/14] Switched green and blue and only color completed when either active or failed is colored --- .../main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index d9bdd2baf7eb6..7271b2baca9b4 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -156,7 +156,7 @@ private[ui] class ExecutorsPage( 0) { - "background:hsla(120, 100%, 25%, " + activeTasksAlpha + ");color:white" + "background:hsla(240, 100%, 50%, " + activeTasksAlpha + ");color:white" } else { "" } @@ -169,8 +169,8 @@ private[ui] class ExecutorsPage( } }>{info.failedTasks} 0) { - "background:hsla(240, 100%, 50%, " + completedTasksAlpha + ");color:white" + if (info.completedTasks > 0 && (info.activeTasks > 0 || info.failedTasks > 0)) { + "background:hsla(120, 100%, 25%, " + completedTasksAlpha + ");color:white" } else { "" } From 31d6a1ecc8f6a860af46897b492f0d5cc58d1219 Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Mon, 21 Dec 2015 15:28:59 -0800 Subject: [PATCH 07/14] Updated MiMa exclude for new version --- project/MimaExcludes.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 3ff4218ccb1c6..e70b329f5508b 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -168,6 +168,9 @@ object MimaExcludes { // SPARK-3580 Add getNumPartitions method to JavaRDD ProblemFilters.exclude[MissingMethodProblem]( "org.apache.spark.api.java.JavaRDDLike.getNumPartitions") + ) ++ Seq( + // SPARK-12149 Added new fields to ExecutorSummary + ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.status.api.v1.ExecutorSummary.this") ) ++ // SPARK-11314: YARN backend moved to yarn sub-module and MiMA complains even though it's a // private class. From 5bcc2983a62c8b3c311a3df65c87c35b5ad1f150 Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Wed, 6 Jan 2016 10:49:49 -0800 Subject: [PATCH 08/14] Reverted completed to always colored when greater than zero --- .../src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index 7271b2baca9b4..ac0d4a2da7826 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -169,7 +169,7 @@ private[ui] class ExecutorsPage( } }>{info.failedTasks} 0 && (info.activeTasks > 0 || info.failedTasks > 0)) { + if (info.completedTasks > 0) { "background:hsla(120, 100%, 25%, " + completedTasksAlpha + ");color:white" } else { "" From 70ab74829d341af63e1d3b05ecc537d99f2243be Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Tue, 12 Jan 2016 20:55:30 -0800 Subject: [PATCH 09/14] Addressed comments - added GC Time and a Task Time tool tip --- core/src/main/scala/org/apache/spark/ui/ToolTips.scala | 3 +++ .../scala/org/apache/spark/ui/exec/ExecutorsPage.scala | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/ToolTips.scala b/core/src/main/scala/org/apache/spark/ui/ToolTips.scala index cb122eaed83d1..2d2d80be4aabe 100644 --- a/core/src/main/scala/org/apache/spark/ui/ToolTips.scala +++ b/core/src/main/scala/org/apache/spark/ui/ToolTips.scala @@ -87,4 +87,7 @@ private[spark] object ToolTips { multiple operations (e.g. two map() functions) if they can be pipelined. Some operations also create multiple RDDs internally. Cached RDDs are shown in green. """ + + val TASK_TIME = + "Shaded red when garbage collection (GC) time is over 10% of task time" } diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index ac0d4a2da7826..4f934c2f91fc5 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -73,7 +73,7 @@ private[ui] class ExecutorsPage( Failed Tasks Complete Tasks Total Tasks - Task Time + Task Time (GC Time) Input Shuffle Read @@ -114,6 +114,9 @@ private[ui] class ExecutorsPage( /** Render an HTML row representing an executor */ private def execRow(info: ExecutorSummary, logsExist: Boolean): Seq[Node] = { + // When GCTimePercent is edited change ToolTips.TASK_TIME to match + val GCTimePercent = 0.1 + val maximumMemory = info.maxMemory val memoryUsed = info.memoryUsed val diskUsed = info.diskUsed @@ -177,14 +180,15 @@ private[ui] class ExecutorsPage( }>{info.completedTasks} {info.totalTasks} info.totalDuration) { + // Red if GC time over GCTimePercent of total time + if (info.totalGCTime > GCTimePercent * info.totalDuration) { "background:hsla(0, 100%, 50%, " + totalDurationAlpha + ");color:white" } else { "" } }> {Utils.msDurationToString(info.totalDuration)} + ({Utils.msDurationToString(info.totalGCTime)}) {Utils.bytesToString(info.totalInputBytes)} From 4294d3f4639be49c8031e35b41c8b59ee241b2fc Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Thu, 14 Jan 2016 14:51:02 -0800 Subject: [PATCH 10/14] Replaced totalCores with maxTasks to include spark.task.cpus conf use case --- .../scala/org/apache/spark/status/api/v1/api.scala | 2 +- core/src/main/scala/org/apache/spark/ui/SparkUI.scala | 2 +- .../scala/org/apache/spark/ui/exec/ExecutorsPage.scala | 8 ++++---- .../scala/org/apache/spark/ui/exec/ExecutorsTab.scala | 10 ++++++---- .../executor_list_json_expectation.json | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index 43a915d5825cc..11010ba5d14cf 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -55,7 +55,7 @@ class ExecutorSummary private[spark]( val rddBlocks: Int, val memoryUsed: Long, val diskUsed: Long, - val totalCores: Int, + val maxTasks: Int, val activeTasks: Int, val failedTasks: Int, val completedTasks: Int, diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala index e319937702f23..873288ed0ac09 100644 --- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala @@ -194,7 +194,7 @@ private[spark] object SparkUI { val environmentListener = new EnvironmentListener val storageStatusListener = new StorageStatusListener - val executorsListener = new ExecutorsListener(storageStatusListener) + val executorsListener = new ExecutorsListener(storageStatusListener, conf) val storageListener = new StorageListener(storageStatusListener) val operationGraphListener = new RDDOperationGraphListener(conf) diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index 4f934c2f91fc5..23e3f859a2d8c 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -124,8 +124,8 @@ private[ui] class ExecutorsPage( // Determine Color Opacity from 0.5-1 // activeTasks range from 0 to all cores val activeTasksAlpha = - if (info.totalCores > 0) { - (info.activeTasks.toDouble / info.totalCores) * 0.5 + 0.5 + if (info.maxTasks > 0) { + (info.activeTasks.toDouble / info.maxTasks) * 0.5 + 0.5 } else { 1 } @@ -239,7 +239,7 @@ private[spark] object ExecutorsPage { val memUsed = status.memUsed val maxMem = status.maxMem val diskUsed = status.diskUsed - val totalCores = listener.executorToTotalCores.getOrElse(execId, 0) + val maxTasks = listener.executorToTasksMax.getOrElse(execId, 0) val activeTasks = listener.executorToTasksActive.getOrElse(execId, 0) val failedTasks = listener.executorToTasksFailed.getOrElse(execId, 0) val completedTasks = listener.executorToTasksComplete.getOrElse(execId, 0) @@ -257,7 +257,7 @@ private[spark] object ExecutorsPage { rddBlocks, memUsed, diskUsed, - totalCores, + maxTasks, activeTasks, failedTasks, completedTasks, diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala index 3b95740b080ae..ff34d9bf95677 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala @@ -19,7 +19,7 @@ package org.apache.spark.ui.exec import scala.collection.mutable.HashMap -import org.apache.spark.{ExceptionFailure, Resubmitted, SparkContext} +import org.apache.spark.{SparkConf, ExceptionFailure, Resubmitted, SparkContext} import org.apache.spark.annotation.DeveloperApi import org.apache.spark.scheduler._ import org.apache.spark.storage.{StorageStatus, StorageStatusListener} @@ -43,8 +43,9 @@ private[ui] class ExecutorsTab(parent: SparkUI) extends SparkUITab(parent, "exec * A SparkListener that prepares information to be displayed on the ExecutorsTab */ @DeveloperApi -class ExecutorsListener(storageStatusListener: StorageStatusListener) extends SparkListener { - val executorToTotalCores = HashMap[String, Int]() +class ExecutorsListener(storageStatusListener: StorageStatusListener, conf: SparkConf) + extends SparkListener { + val executorToTasksMax = HashMap[String, Int]() val executorToTasksActive = HashMap[String, Int]() val executorToTasksComplete = HashMap[String, Int]() val executorToTasksFailed = HashMap[String, Int]() @@ -64,7 +65,8 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener) extends Sp override def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit = synchronized { val eid = executorAdded.executorId executorToLogUrls(eid) = executorAdded.executorInfo.logUrlMap - executorToTotalCores(eid) = executorAdded.executorInfo.totalCores + executorToTasksMax(eid) = + executorAdded.executorInfo.totalCores / conf.getInt("spark.task.cpus", 1) executorIdToData(eid) = ExecutorUIData(executorAdded.time) } diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json index f407bfa951739..94f8aeac55b5d 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json @@ -4,7 +4,7 @@ "rddBlocks" : 8, "memoryUsed" : 28000128, "diskUsed" : 0, - "totalCores" : 0, + "maxTasks" : 0, "activeTasks" : 0, "failedTasks" : 1, "completedTasks" : 31, From 07ba26ad45e45030ffd62e06e501b3bb6abe5310 Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Thu, 14 Jan 2016 16:03:13 -0800 Subject: [PATCH 11/14] Fixed style issue --- core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala index ff34d9bf95677..504793b8ad198 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala @@ -19,7 +19,7 @@ package org.apache.spark.ui.exec import scala.collection.mutable.HashMap -import org.apache.spark.{SparkConf, ExceptionFailure, Resubmitted, SparkContext} +import org.apache.spark.{ExceptionFailure, Resubmitted, SparkConf, SparkContext} import org.apache.spark.annotation.DeveloperApi import org.apache.spark.scheduler._ import org.apache.spark.storage.{StorageStatus, StorageStatusListener} From 05f957a7274fbbb918c0765d8608a3ee820c1fac Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Fri, 15 Jan 2016 18:06:23 -0800 Subject: [PATCH 12/14] Added color and GC time to new totals table --- .../apache/spark/ui/exec/ExecutorsPage.scala | 153 ++++++++++-------- 1 file changed, 83 insertions(+), 70 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index f1db1bd2bfa5e..ce3ca0fd36c21 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -50,6 +50,8 @@ private[ui] class ExecutorsPage( threadDumpEnabled: Boolean) extends WebUIPage("") { private val listener = parent.listener + // When GCTimePercent is edited change ToolTips.TASK_TIME to match + private val GCTimePercent = 0.1 def render(request: HttpServletRequest): Seq[Node] = { val (storageStatusList, execInfo) = listener.synchronized { @@ -115,38 +117,10 @@ private[ui] class ExecutorsPage( /** Render an HTML row representing an executor */ private def execRow(info: ExecutorSummary, logsExist: Boolean): Seq[Node] = { - // When GCTimePercent is edited change ToolTips.TASK_TIME to match - val GCTimePercent = 0.1 - val maximumMemory = info.maxMemory val memoryUsed = info.memoryUsed val diskUsed = info.diskUsed - // Determine Color Opacity from 0.5-1 - // activeTasks range from 0 to all cores - val activeTasksAlpha = - if (info.maxTasks > 0) { - (info.activeTasks.toDouble / info.maxTasks) * 0.5 + 0.5 - } else { - 1 - } - // failedTasks range max at 10% failure, alpha max = 1 - // completedTasks range ignores 90% of tasks - val (failedTasksAlpha, completedTasksAlpha) = - if (info.totalTasks > 0) { - (math.min(10 * info.failedTasks.toDouble / info.totalTasks, 1) * 0.5 + 0.5, - math.max(((10 * info.completedTasks.toDouble / info.totalTasks) - 9) * 0.5 + 0.5, 0.5)) - } else { - (1, 1) - } - // totalDuration range from 0 to 50% GC time, alpha max = 1 - val totalDurationAlpha = - if (info.totalDuration > 0) { - math.min(info.totalGCTime.toDouble / info.totalDuration + 0.5, 1) - } else { - 1 - } - {info.id} {info.hostPort} @@ -158,39 +132,8 @@ private[ui] class ExecutorsPage( {Utils.bytesToString(diskUsed)} - 0) { - "background:hsla(240, 100%, 50%, " + activeTasksAlpha + ");color:white" - } else { - "" - } - }>{info.activeTasks} - 0) { - "background:hsla(0, 100%, 50%, " + failedTasksAlpha + ");color:white" - } else { - "" - } - }>{info.failedTasks} - 0) { - "background:hsla(120, 100%, 25%, " + completedTasksAlpha + ");color:white" - } else { - "" - } - }>{info.completedTasks} - {info.totalTasks} - GCTimePercent * info.totalDuration) { - "background:hsla(0, 100%, 50%, " + totalDurationAlpha + ");color:white" - } else { - "" - } - }> - {Utils.msDurationToString(info.totalDuration)} - ({Utils.msDurationToString(info.totalGCTime)}) - + {coloredData(info.maxTasks, info.activeTasks, info.failedTasks, info.completedTasks, + info.totalTasks, info.totalDuration, info.totalGCTime)} {Utils.bytesToString(info.totalInputBytes)} @@ -232,7 +175,6 @@ private[ui] class ExecutorsPage( val maximumMemory = execInfo.map(_.maxMemory).sum val memoryUsed = execInfo.map(_.memoryUsed).sum val diskUsed = execInfo.map(_.diskUsed).sum - val totalDuration = execInfo.map(_.totalDuration).sum val totalInputBytes = execInfo.map(_.totalInputBytes).sum val totalShuffleRead = execInfo.map(_.totalShuffleRead).sum val totalShuffleWrite = execInfo.map(_.totalShuffleWrite).sum @@ -247,13 +189,13 @@ private[ui] class ExecutorsPage( {Utils.bytesToString(diskUsed)} - {execInfo.map(_.activeTasks).sum} - {execInfo.map(_.failedTasks).sum} - {execInfo.map(_.completedTasks).sum} - {execInfo.map(_.totalTasks).sum} - - {Utils.msDurationToString(totalDuration)} - + {coloredData(execInfo.map(_.maxTasks).sum, + execInfo.map(_.activeTasks).sum, + execInfo.map(_.failedTasks).sum, + execInfo.map(_.completedTasks).sum, + execInfo.map(_.totalTasks).sum, + execInfo.map(_.totalDuration).sum, + execInfo.map(_.totalGCTime).sum)} {Utils.bytesToString(totalInputBytes)} @@ -274,7 +216,7 @@ private[ui] class ExecutorsPage( Failed Tasks Complete Tasks Total Tasks - Task Time + Task Time (GC Time) Input Shuffle Read @@ -288,6 +230,77 @@ private[ui] class ExecutorsPage( } + + private def coloredData(maxTasks: Int, + activeTasks: Int, + failedTasks: Int, + completedTasks: Int, + totalTasks: Int, + totalDuration: Long, + totalGCTime: Long): + Seq[Node] = { + // Determine Color Opacity from 0.5-1 + // activeTasks range from 0 to all cores + val activeTasksAlpha = + if (maxTasks > 0) { + (activeTasks.toDouble / maxTasks) * 0.5 + 0.5 + } else { + 1 + } + // failedTasks range max at 10% failure, alpha max = 1 + // completedTasks range ignores 90% of tasks + val (failedTasksAlpha, completedTasksAlpha) = + if (totalTasks > 0) { + (math.min (10 * failedTasks.toDouble / totalTasks, 1) * 0.5 + 0.5, + math.max (((10 * completedTasks.toDouble / totalTasks) - 9) * 0.5 + 0.5, 0.5) ) + } else { + (1, 1) + } + // totalDuration range from 0 to 50% GC time, alpha max = 1 + val totalDurationAlpha = + if (totalDuration > 0) { + math.min (totalGCTime.toDouble / totalDuration + 0.5, 1) + } else { + 1 + } + + val tableData = + 0) { + "background:hsla(240, 100%, 50%, " + activeTasksAlpha + ");color:white" + } else { + "" + } + }>{activeTasks} + 0) { + "background:hsla(0, 100%, 50%, " + failedTasksAlpha + ");color:white" + } else { + "" + } + }>{failedTasks} + 0) { + "background:hsla(120, 100%, 25%, " + completedTasksAlpha + ");color:white" + } else { + "" + } + }>{completedTasks} + {totalTasks} + GCTimePercent * totalDuration) { + "background:hsla(0, 100%, 50%, " + totalDurationAlpha + ");color:white" + } else { + "" + } + }> + {Utils.msDurationToString(totalDuration)} + ({Utils.msDurationToString(totalGCTime)}) + ; + + tableData + } } private[spark] object ExecutorsPage { From 2f54a333f8d6f9250132679a61783550de5fe34e Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Tue, 19 Jan 2016 14:53:45 -0800 Subject: [PATCH 13/14] Removed Completed Tasks coloring --- .../apache/spark/ui/exec/ExecutorsPage.scala | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index ce3ca0fd36c21..4395c7f0c7e68 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -120,7 +120,6 @@ private[ui] class ExecutorsPage( val maximumMemory = info.maxMemory val memoryUsed = info.memoryUsed val diskUsed = info.diskUsed - {info.id} {info.hostPort} @@ -132,7 +131,7 @@ private[ui] class ExecutorsPage( {Utils.bytesToString(diskUsed)} - {coloredData(info.maxTasks, info.activeTasks, info.failedTasks, info.completedTasks, + {taskData(info.maxTasks, info.activeTasks, info.failedTasks, info.completedTasks, info.totalTasks, info.totalDuration, info.totalGCTime)} {Utils.bytesToString(info.totalInputBytes)} @@ -189,7 +188,7 @@ private[ui] class ExecutorsPage( {Utils.bytesToString(diskUsed)} - {coloredData(execInfo.map(_.maxTasks).sum, + {taskData(execInfo.map(_.maxTasks).sum, execInfo.map(_.activeTasks).sum, execInfo.map(_.failedTasks).sum, execInfo.map(_.completedTasks).sum, @@ -231,7 +230,7 @@ private[ui] class ExecutorsPage( } - private def coloredData(maxTasks: Int, + private def taskData(maxTasks: Int, activeTasks: Int, failedTasks: Int, completedTasks: Int, @@ -240,7 +239,7 @@ private[ui] class ExecutorsPage( totalGCTime: Long): Seq[Node] = { // Determine Color Opacity from 0.5-1 - // activeTasks range from 0 to all cores + // activeTasks range from 0 to maxTasks val activeTasksAlpha = if (maxTasks > 0) { (activeTasks.toDouble / maxTasks) * 0.5 + 0.5 @@ -248,13 +247,11 @@ private[ui] class ExecutorsPage( 1 } // failedTasks range max at 10% failure, alpha max = 1 - // completedTasks range ignores 90% of tasks - val (failedTasksAlpha, completedTasksAlpha) = + val failedTasksAlpha = if (totalTasks > 0) { - (math.min (10 * failedTasks.toDouble / totalTasks, 1) * 0.5 + 0.5, - math.max (((10 * completedTasks.toDouble / totalTasks) - 9) * 0.5 + 0.5, 0.5) ) + math.min (10 * failedTasks.toDouble / totalTasks, 1) * 0.5 + 0.5 } else { - (1, 1) + 1 } // totalDuration range from 0 to 50% GC time, alpha max = 1 val totalDurationAlpha = @@ -279,13 +276,7 @@ private[ui] class ExecutorsPage( "" } }>{failedTasks} - 0) { - "background:hsla(120, 100%, 25%, " + completedTasksAlpha + ");color:white" - } else { - "" - } - }>{completedTasks} + {completedTasks} {totalTasks} Date: Mon, 25 Jan 2016 08:53:14 -0800 Subject: [PATCH 14/14] Fixed some style issues --- .../apache/spark/ui/exec/ExecutorsPage.scala | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index 4395c7f0c7e68..e36b96b3e6978 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -230,13 +230,14 @@ private[ui] class ExecutorsPage( } - private def taskData(maxTasks: Int, - activeTasks: Int, - failedTasks: Int, - completedTasks: Int, - totalTasks: Int, - totalDuration: Long, - totalGCTime: Long): + private def taskData( + maxTasks: Int, + activeTasks: Int, + failedTasks: Int, + completedTasks: Int, + totalTasks: Int, + totalDuration: Long, + totalGCTime: Long): Seq[Node] = { // Determine Color Opacity from 0.5-1 // activeTasks range from 0 to maxTasks @@ -249,14 +250,14 @@ private[ui] class ExecutorsPage( // failedTasks range max at 10% failure, alpha max = 1 val failedTasksAlpha = if (totalTasks > 0) { - math.min (10 * failedTasks.toDouble / totalTasks, 1) * 0.5 + 0.5 + math.min(10 * failedTasks.toDouble / totalTasks, 1) * 0.5 + 0.5 } else { 1 } // totalDuration range from 0 to 50% GC time, alpha max = 1 val totalDurationAlpha = if (totalDuration > 0) { - math.min (totalGCTime.toDouble / totalDuration + 0.5, 1) + math.min(totalGCTime.toDouble / totalDuration + 0.5, 1) } else { 1 }