diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationInfo.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationInfo.scala deleted file mode 100644 index 6851bf72077c0..0000000000000 --- a/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationInfo.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.status.api.v1 - -import java.util.Date - -case class ApplicationInfo( - id: String, - name: String, - startTime: Date, - endTime: Date, - sparkUser: String, - completed: Boolean = false) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ExecutorStageSummary.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ExecutorStageSummary.scala deleted file mode 100644 index a85ffba6e0a2e..0000000000000 --- a/core/src/main/scala/org/apache/spark/status/api/v1/ExecutorStageSummary.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.status.api.v1 - -case class ExecutorStageSummary( - taskTime : Long, - failedTasks : Int, - succeededTasks : Int, - inputBytes : Long, - outputBytes : Long, - shuffleRead : Long, - shuffleWrite : Long, - memoryBytesSpilled : Long, - diskBytesSpilled : Long -) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ExecutorSummary.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ExecutorSummary.scala deleted file mode 100644 index 58f473935dcb0..0000000000000 --- a/core/src/main/scala/org/apache/spark/status/api/v1/ExecutorSummary.scala +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.status.api.v1 - -case class ExecutorSummary( - id: String, - hostPort: String, - rddBlocks: Int, - memoryUsed: Long, - diskUsed: Long, - activeTasks: Int, - failedTasks: Int, - completedTasks: Int, - totalTasks: Int, - totalDuration: Long, - totalInputBytes: Long, - totalShuffleRead: Long, - totalShuffleWrite: Long, - maxMemory: Long, - executorLogs: Map[String, String] -) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/JobData.scala b/core/src/main/scala/org/apache/spark/status/api/v1/JobData.scala deleted file mode 100644 index 683f446a1ff54..0000000000000 --- a/core/src/main/scala/org/apache/spark/status/api/v1/JobData.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.status.api.v1 - -import java.util.Date - -import org.apache.spark.JobExecutionStatus - -case class JobData( - jobId: Int, - name: String, - description: Option[String], - submissionTime: Option[Date], - completionTime: Option[Date], - stageIds: Seq[Int], - jobGroup: Option[String], - status: JobExecutionStatus, - numTasks: Int, - numActiveTasks: Int, - numCompletedTasks: Int, - numSkippedTasks: Int, - numFailedTasks: Int, - numActiveStages: Int, - numCompletedStages: Int, - numSkippedStages: Int, - numFailedStages: Int -) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/RDDStorageInfo.scala b/core/src/main/scala/org/apache/spark/status/api/v1/RDDStorageInfo.scala deleted file mode 100644 index 59d99e8374d12..0000000000000 --- a/core/src/main/scala/org/apache/spark/status/api/v1/RDDStorageInfo.scala +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.status.api.v1 - -// Q: should Tachyon size go in here as well? currently the UI only shows it on the overall storage -// page ... does anybody pay attention to it? -case class RDDStorageInfo( - id: Int, - name: String, - numPartitions: Int, - numCachedPartitions: Int, - storageLevel: String, - memoryUsed: Long, - diskUsed: Long, - dataDistribution: Option[Seq[RDDDataDistribution]], - partitions: Option[Seq[RDDPartitionInfo]] -) - -case class RDDDataDistribution( - address: String, - memoryUsed: Long, - memoryRemaining: Long, - diskUsed: Long -) - -case class RDDPartitionInfo( - blockName: String, - storageLevel: String, - memoryUsed: Long, - diskUsed: Long, - executors: Seq[String] -) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/StageData.scala b/core/src/main/scala/org/apache/spark/status/api/v1/StageData.scala deleted file mode 100644 index df1a6f0d599bd..0000000000000 --- a/core/src/main/scala/org/apache/spark/status/api/v1/StageData.scala +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.status.api.v1 - -import scala.collection.Map - -import org.apache.spark.status.api.StageStatus - -case class StageData( - status: StageStatus, - stageId: Int, - numActiveTasks: Int , - numCompleteTasks: Int, - numFailedTasks: Int, - - executorRunTime: Long, - - inputBytes: Long, - inputRecords: Long, - outputBytes: Long, - outputRecords: Long, - shuffleReadBytes: Long, - shuffleReadRecords: Long, - shuffleWriteBytes: Long, - shuffleWriteRecords: Long, - memoryBytesSpilled: Long, - diskBytesSpilled: Long, - - name: String, - details: String, - schedulingPool: String, - - //TODO what to do about accumulables? - tasks: Option[Map[Long, TaskData]], - executorSummary:Option[Map[String,ExecutorStageSummary]] -) - diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/TaskData.scala b/core/src/main/scala/org/apache/spark/status/api/v1/TaskData.scala deleted file mode 100644 index b70cae87f9cd5..0000000000000 --- a/core/src/main/scala/org/apache/spark/status/api/v1/TaskData.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.status.api.v1 - -import java.util.Date - -case class TaskData( - taskId: Long, - index: Int, - attempt: Int, - launchTime: Date, - executorId: String, - host: String, - taskLocality: String, - speculative: Boolean, - errorMessage: Option[String] = None, - taskMetrics: Option[TaskMetrics] = None -) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/TaskMetrics.scala b/core/src/main/scala/org/apache/spark/status/api/v1/TaskMetrics.scala deleted file mode 100644 index acae428f3139e..0000000000000 --- a/core/src/main/scala/org/apache/spark/status/api/v1/TaskMetrics.scala +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.status.api.v1 - -case class TaskMetrics( - executorDeserializeTime: Long, - executorRunTime: Long, - resultSize: Long, - jvmGcTime: Long, - resultSerializationTime: Long, - memoryBytesSpilled: Long, - diskBytesSpilled: Long, - inputMetrics: Option[InputMetrics], - outputMetrics: Option[OutputMetrics], - shuffleReadMetrics: Option[ShuffleReadMetrics], - shuffleWriteMetrics: Option[ShuffleWriteMetrics] -) - -case class InputMetrics( - bytesRead: Long, - recordsRead: Long -) - -case class OutputMetrics( - bytesWritten: Long, - recordsWritten: Long -) - -case class ShuffleReadMetrics( - remoteBlocksFetched: Int, - localBlocksFetched: Int, - fetchWaitTime: Long, - remoteBytesRead: Long, - totalBlocksFetched: Int, - recordsRead: Long -) - -case class ShuffleWriteMetrics( - bytesWritten: Long, - writeTime: Long, - recordsWritten: Long -) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala new file mode 100644 index 0000000000000..9001207ac737b --- /dev/null +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.status.api.v1 + +import java.util.Date + +import scala.collection.Map + +import org.apache.spark.JobExecutionStatus +import org.apache.spark.status.api.StageStatus + +case class ApplicationInfo( + id: String, + name: String, + startTime: Date, + endTime: Date, + sparkUser: String, + completed: Boolean = false +) + +case class ExecutorStageSummary( + taskTime : Long, + failedTasks : Int, + succeededTasks : Int, + inputBytes : Long, + outputBytes : Long, + shuffleRead : Long, + shuffleWrite : Long, + memoryBytesSpilled : Long, + diskBytesSpilled : Long +) + +case class ExecutorSummary( + id: String, + hostPort: String, + rddBlocks: Int, + memoryUsed: Long, + diskUsed: Long, + activeTasks: Int, + failedTasks: Int, + completedTasks: Int, + totalTasks: Int, + totalDuration: Long, + totalInputBytes: Long, + totalShuffleRead: Long, + totalShuffleWrite: Long, + maxMemory: Long, + executorLogs: Map[String, String] +) + +case class JobData( + jobId: Int, + name: String, + description: Option[String], + submissionTime: Option[Date], + completionTime: Option[Date], + stageIds: Seq[Int], + jobGroup: Option[String], + status: JobExecutionStatus, + numTasks: Int, + numActiveTasks: Int, + numCompletedTasks: Int, + numSkippedTasks: Int, + numFailedTasks: Int, + numActiveStages: Int, + numCompletedStages: Int, + numSkippedStages: Int, + numFailedStages: Int +) + +// Q: should Tachyon size go in here as well? currently the UI only shows it on the overall storage +// page ... does anybody pay attention to it? +case class RDDStorageInfo( + id: Int, + name: String, + numPartitions: Int, + numCachedPartitions: Int, + storageLevel: String, + memoryUsed: Long, + diskUsed: Long, + dataDistribution: Option[Seq[RDDDataDistribution]], + partitions: Option[Seq[RDDPartitionInfo]] +) + +case class RDDDataDistribution( + address: String, + memoryUsed: Long, + memoryRemaining: Long, + diskUsed: Long +) + +case class RDDPartitionInfo( + blockName: String, + storageLevel: String, + memoryUsed: Long, + diskUsed: Long, + executors: Seq[String] +) + +case class StageData( + status: StageStatus, + stageId: Int, + numActiveTasks: Int , + numCompleteTasks: Int, + numFailedTasks: Int, + + executorRunTime: Long, + + inputBytes: Long, + inputRecords: Long, + outputBytes: Long, + outputRecords: Long, + shuffleReadBytes: Long, + shuffleReadRecords: Long, + shuffleWriteBytes: Long, + shuffleWriteRecords: Long, + memoryBytesSpilled: Long, + diskBytesSpilled: Long, + + name: String, + details: String, + schedulingPool: String, + + //TODO what to do about accumulables? + tasks: Option[Map[Long, TaskData]], + executorSummary:Option[Map[String,ExecutorStageSummary]] +) + +case class TaskData( + taskId: Long, + index: Int, + attempt: Int, + launchTime: Date, + executorId: String, + host: String, + taskLocality: String, + speculative: Boolean, + errorMessage: Option[String] = None, + taskMetrics: Option[TaskMetrics] = None +) + +case class TaskMetrics( + executorDeserializeTime: Long, + executorRunTime: Long, + resultSize: Long, + jvmGcTime: Long, + resultSerializationTime: Long, + memoryBytesSpilled: Long, + diskBytesSpilled: Long, + inputMetrics: Option[InputMetrics], + outputMetrics: Option[OutputMetrics], + shuffleReadMetrics: Option[ShuffleReadMetrics], + shuffleWriteMetrics: Option[ShuffleWriteMetrics] +) + +case class InputMetrics( + bytesRead: Long, + recordsRead: Long +) + +case class OutputMetrics( + bytesWritten: Long, + recordsWritten: Long +) + +case class ShuffleReadMetrics( + remoteBlocksFetched: Int, + localBlocksFetched: Int, + fetchWaitTime: Long, + remoteBytesRead: Long, + totalBlocksFetched: Int, + recordsRead: Long +) + +case class ShuffleWriteMetrics( + bytesWritten: Long, + writeTime: Long, + recordsWritten: Long +)