| {rdd.id} |
-
{rdd.name}
diff --git a/core/src/main/scala/org/apache/spark/util/CompletionIterator.scala b/core/src/main/scala/org/apache/spark/util/CompletionIterator.scala
index 21acaa95c5645..f4d6c7a28d2e4 100644
--- a/core/src/main/scala/org/apache/spark/util/CompletionIterator.scala
+++ b/core/src/main/scala/org/apache/spark/util/CompletionIterator.scala
@@ -25,11 +25,14 @@ private[spark]
abstract class CompletionIterator[ +A, +I <: Iterator[A]](sub: I) extends Iterator[A] {
private[this] var completed = false
- def next(): A = sub.next()
+ private[this] var iter = sub
+ def next(): A = iter.next()
def hasNext: Boolean = {
- val r = sub.hasNext
+ val r = iter.hasNext
if (!r && !completed) {
completed = true
+ // reassign to release resources of highly resource consuming iterators early
+ iter = Iterator.empty.asInstanceOf[I]
completion()
}
r
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 93b5826f8a74b..227c9e734f0af 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -31,7 +31,6 @@ import java.security.SecureRandom
import java.util.{Locale, Properties, Random, UUID}
import java.util.concurrent._
import java.util.concurrent.TimeUnit.NANOSECONDS
-import java.util.concurrent.atomic.AtomicBoolean
import java.util.zip.GZIPInputStream
import scala.annotation.tailrec
@@ -93,53 +92,6 @@ private[spark] object Utils extends Logging {
private val MAX_DIR_CREATION_ATTEMPTS: Int = 10
@volatile private var localRootDirs: Array[String] = null
- /**
- * The performance overhead of creating and logging strings for wide schemas can be large. To
- * limit the impact, we bound the number of fields to include by default. This can be overridden
- * by setting the 'spark.debug.maxToStringFields' conf in SparkEnv.
- */
- val DEFAULT_MAX_TO_STRING_FIELDS = 25
-
- private[spark] def maxNumToStringFields = {
- if (SparkEnv.get != null) {
- SparkEnv.get.conf.getInt("spark.debug.maxToStringFields", DEFAULT_MAX_TO_STRING_FIELDS)
- } else {
- DEFAULT_MAX_TO_STRING_FIELDS
- }
- }
-
- /** Whether we have warned about plan string truncation yet. */
- private val truncationWarningPrinted = new AtomicBoolean(false)
-
- /**
- * Format a sequence with semantics similar to calling .mkString(). Any elements beyond
- * maxNumToStringFields will be dropped and replaced by a "... N more fields" placeholder.
- *
- * @return the trimmed and formatted string.
- */
- def truncatedString[T](
- seq: Seq[T],
- start: String,
- sep: String,
- end: String,
- maxNumFields: Int = maxNumToStringFields): String = {
- if (seq.length > maxNumFields) {
- if (truncationWarningPrinted.compareAndSet(false, true)) {
- logWarning(
- "Truncated the string representation of a plan since it was too large. This " +
- "behavior can be adjusted by setting 'spark.debug.maxToStringFields' in SparkEnv.conf.")
- }
- val numFields = math.max(0, maxNumFields - 1)
- seq.take(numFields).mkString(
- start, sep, sep + "... " + (seq.length - numFields) + " more fields" + end)
- } else {
- seq.mkString(start, sep, end)
- }
- }
-
- /** Shorthand for calling truncatedString() without start or end strings. */
- def truncatedString[T](seq: Seq[T], sep: String): String = truncatedString(seq, "", sep, "")
-
/** Serialize an object using Java serialization */
def serialize[T](o: T): Array[Byte] = {
val bos = new ByteArrayOutputStream()
@@ -2328,7 +2280,12 @@ private[spark] object Utils extends Logging {
* configure a new log4j level
*/
def setLogLevel(l: org.apache.log4j.Level) {
- org.apache.log4j.Logger.getRootLogger().setLevel(l)
+ val rootLogger = org.apache.log4j.Logger.getRootLogger()
+ rootLogger.setLevel(l)
+ rootLogger.getAllAppenders().asScala.foreach {
+ case ca: org.apache.log4j.ConsoleAppender => ca.setThreshold(l)
+ case _ => // no-op
+ }
}
/**
@@ -2430,7 +2387,8 @@ private[spark] object Utils extends Logging {
"org.apache.spark.security.ShellBasedGroupsMappingProvider")
if (groupProviderClassName != "") {
try {
- val groupMappingServiceProvider = classForName(groupProviderClassName).newInstance.
+ val groupMappingServiceProvider = classForName(groupProviderClassName).
+ getConstructor().newInstance().
asInstanceOf[org.apache.spark.security.GroupMappingServiceProvider]
val currentUserGroups = groupMappingServiceProvider.getGroups(username)
return currentUserGroups
@@ -2863,6 +2821,14 @@ private[spark] object Utils extends Logging {
def stringHalfWidth(str: String): Int = {
if (str == null) 0 else str.length + fullWidthRegex.findAllIn(str).size
}
+
+ def sanitizeDirName(str: String): String = {
+ str.replaceAll("[ :/]", "-").replaceAll("[.${}'\"]", "_").toLowerCase(Locale.ROOT)
+ }
+
+ def isClientMode(conf: SparkConf): Boolean = {
+ "client".equals(conf.get(SparkLauncher.DEPLOY_MODE, "client"))
+ }
}
private[util] object CallerContext extends Logging {
diff --git a/core/src/main/scala/org/apache/spark/util/VersionUtils.scala b/core/src/main/scala/org/apache/spark/util/VersionUtils.scala
index 828153b868420..c0f8866dd58dc 100644
--- a/core/src/main/scala/org/apache/spark/util/VersionUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/VersionUtils.scala
@@ -23,6 +23,7 @@ package org.apache.spark.util
private[spark] object VersionUtils {
private val majorMinorRegex = """^(\d+)\.(\d+)(\..*)?$""".r
+ private val shortVersionRegex = """^(\d+\.\d+\.\d+)(.*)?$""".r
/**
* Given a Spark version string, return the major version number.
@@ -36,6 +37,19 @@ private[spark] object VersionUtils {
*/
def minorVersion(sparkVersion: String): Int = majorMinorVersion(sparkVersion)._2
+ /**
+ * Given a Spark version string, return the short version string.
+ * E.g., for 3.0.0-SNAPSHOT, return '3.0.0'.
+ */
+ def shortVersion(sparkVersion: String): String = {
+ shortVersionRegex.findFirstMatchIn(sparkVersion) match {
+ case Some(m) => m.group(1)
+ case None =>
+ throw new IllegalArgumentException(s"Spark tried to parse '$sparkVersion' as a Spark" +
+ s" version string, but it could not find the major/minor/maintenance version numbers.")
+ }
+ }
+
/**
* Given a Spark version string, return the (major version number, minor version number).
* E.g., for 2.0.1-SNAPSHOT, return (2, 0).
diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
index b159200d79222..46279e79d78db 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
@@ -727,9 +727,10 @@ private[spark] class ExternalSorter[K, V, C](
spills.clear()
forceSpillFiles.foreach(s => s.file.delete())
forceSpillFiles.clear()
- if (map != null || buffer != null) {
+ if (map != null || buffer != null || readingIterator != null) {
map = null // So that the memory can be garbage-collected
buffer = null // So that the memory can be garbage-collected
+ readingIterator = null // So that the memory can be garbage-collected
releaseMemory()
}
}
@@ -793,8 +794,8 @@ private[spark] class ExternalSorter[K, V, C](
def nextPartition(): Int = cur._1._1
}
- logInfo(s"Task ${context.taskAttemptId} force spilling in-memory map to disk and " +
- s" it will release ${org.apache.spark.util.Utils.bytesToString(getUsed())} memory")
+ logInfo(s"Task ${TaskContext.get().taskAttemptId} force spilling in-memory map to disk " +
+ s"and it will release ${org.apache.spark.util.Utils.bytesToString(getUsed())} memory")
val spillFile = spillMemoryIteratorToDisk(inMemoryIterator)
forceSpillFiles += spillFile
val spillReader = new SpillReader(spillFile)
diff --git a/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala b/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala
index 870830fff4c3e..128d6ff8cd746 100644
--- a/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala
+++ b/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala
@@ -222,7 +222,8 @@ private[spark] class ChunkedByteBufferInputStream(
dispose: Boolean)
extends InputStream {
- private[this] var chunks = chunkedByteBuffer.getChunks().iterator
+ // Filter out empty chunks since `read()` assumes all chunks are non-empty.
+ private[this] var chunks = chunkedByteBuffer.getChunks().filter(_.hasRemaining).iterator
private[this] var currentChunk: ByteBuffer = {
if (chunks.hasNext) {
chunks.next()
diff --git a/core/src/main/scala/org/apache/spark/util/logging/DriverLogger.scala b/core/src/main/scala/org/apache/spark/util/logging/DriverLogger.scala
new file mode 100644
index 0000000000000..bea18a3df4783
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/util/logging/DriverLogger.scala
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util.logging
+
+import java.io._
+import java.util.concurrent.{ScheduledExecutorService, TimeUnit}
+
+import org.apache.commons.io.FileUtils
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FileSystem, FSDataOutputStream, Path}
+import org.apache.hadoop.fs.permission.FsPermission
+import org.apache.log4j.{FileAppender => Log4jFileAppender, _}
+
+import org.apache.spark.SparkConf
+import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config._
+import org.apache.spark.network.util.JavaUtils
+import org.apache.spark.util.{ThreadUtils, Utils}
+
+private[spark] class DriverLogger(conf: SparkConf) extends Logging {
+
+ private val UPLOAD_CHUNK_SIZE = 1024 * 1024
+ private val UPLOAD_INTERVAL_IN_SECS = 5
+ private val DEFAULT_LAYOUT = "%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n"
+ private val LOG_FILE_PERMISSIONS = new FsPermission(Integer.parseInt("770", 8).toShort)
+
+ private var localLogFile: String = FileUtils.getFile(
+ Utils.getLocalDir(conf),
+ DriverLogger.DRIVER_LOG_DIR,
+ DriverLogger.DRIVER_LOG_FILE).getAbsolutePath()
+ private var writer: Option[DfsAsyncWriter] = None
+
+ addLogAppender()
+
+ private def addLogAppender(): Unit = {
+ val appenders = LogManager.getRootLogger().getAllAppenders()
+ val layout = if (conf.contains(DRIVER_LOG_LAYOUT)) {
+ new PatternLayout(conf.get(DRIVER_LOG_LAYOUT).get)
+ } else if (appenders.hasMoreElements()) {
+ appenders.nextElement().asInstanceOf[Appender].getLayout()
+ } else {
+ new PatternLayout(DEFAULT_LAYOUT)
+ }
+ val fa = new Log4jFileAppender(layout, localLogFile)
+ fa.setName(DriverLogger.APPENDER_NAME)
+ LogManager.getRootLogger().addAppender(fa)
+ logInfo(s"Added a local log appender at: ${localLogFile}")
+ }
+
+ def startSync(hadoopConf: Configuration): Unit = {
+ try {
+ // Setup a writer which moves the local file to hdfs continuously
+ val appId = Utils.sanitizeDirName(conf.getAppId)
+ writer = Some(new DfsAsyncWriter(appId, hadoopConf))
+ } catch {
+ case e: Exception =>
+ logError(s"Could not persist driver logs to dfs", e)
+ }
+ }
+
+ def stop(): Unit = {
+ try {
+ val fa = LogManager.getRootLogger.getAppender(DriverLogger.APPENDER_NAME)
+ LogManager.getRootLogger().removeAppender(DriverLogger.APPENDER_NAME)
+ Utils.tryLogNonFatalError(fa.close())
+ writer.foreach(_.closeWriter())
+ } catch {
+ case e: Exception =>
+ logError(s"Error in persisting driver logs", e)
+ } finally {
+ Utils.tryLogNonFatalError {
+ JavaUtils.deleteRecursively(FileUtils.getFile(localLogFile).getParentFile())
+ }
+ }
+ }
+
+ // Visible for testing
+ private[spark] class DfsAsyncWriter(appId: String, hadoopConf: Configuration) extends Runnable
+ with Logging {
+
+ private var streamClosed = false
+ private var inStream: InputStream = null
+ private var outputStream: FSDataOutputStream = null
+ private val tmpBuffer = new Array[Byte](UPLOAD_CHUNK_SIZE)
+ private var threadpool: ScheduledExecutorService = _
+ init()
+
+ private def init(): Unit = {
+ val rootDir = conf.get(DRIVER_LOG_DFS_DIR).get
+ val fileSystem: FileSystem = new Path(rootDir).getFileSystem(hadoopConf)
+ if (!fileSystem.exists(new Path(rootDir))) {
+ throw new RuntimeException(s"${rootDir} does not exist." +
+ s" Please create this dir in order to persist driver logs")
+ }
+ val dfsLogFile: String = FileUtils.getFile(rootDir, appId
+ + DriverLogger.DRIVER_LOG_FILE_SUFFIX).getAbsolutePath()
+ try {
+ inStream = new BufferedInputStream(new FileInputStream(localLogFile))
+ outputStream = fileSystem.create(new Path(dfsLogFile), true)
+ fileSystem.setPermission(new Path(dfsLogFile), LOG_FILE_PERMISSIONS)
+ } catch {
+ case e: Exception =>
+ JavaUtils.closeQuietly(inStream)
+ JavaUtils.closeQuietly(outputStream)
+ throw e
+ }
+ threadpool = ThreadUtils.newDaemonSingleThreadScheduledExecutor("dfsSyncThread")
+ threadpool.scheduleWithFixedDelay(this, UPLOAD_INTERVAL_IN_SECS, UPLOAD_INTERVAL_IN_SECS,
+ TimeUnit.SECONDS)
+ logInfo(s"Started driver log file sync to: ${dfsLogFile}")
+ }
+
+ def run(): Unit = {
+ if (streamClosed) {
+ return
+ }
+ try {
+ var remaining = inStream.available()
+ while (remaining > 0) {
+ val read = inStream.read(tmpBuffer, 0, math.min(remaining, UPLOAD_CHUNK_SIZE))
+ outputStream.write(tmpBuffer, 0, read)
+ remaining -= read
+ }
+ outputStream.hflush()
+ } catch {
+ case e: Exception => logError("Failed writing driver logs to dfs", e)
+ }
+ }
+
+ private def close(): Unit = {
+ if (streamClosed) {
+ return
+ }
+ try {
+ // Write all remaining bytes
+ run()
+ } finally {
+ try {
+ streamClosed = true
+ inStream.close()
+ outputStream.close()
+ } catch {
+ case e: Exception =>
+ logError("Error in closing driver log input/output stream", e)
+ }
+ }
+ }
+
+ def closeWriter(): Unit = {
+ try {
+ threadpool.execute(new Runnable() {
+ override def run(): Unit = DfsAsyncWriter.this.close()
+ })
+ threadpool.shutdown()
+ threadpool.awaitTermination(1, TimeUnit.MINUTES)
+ } catch {
+ case e: Exception =>
+ logError("Error in shutting down threadpool", e)
+ }
+ }
+ }
+
+}
+
+private[spark] object DriverLogger extends Logging {
+ val DRIVER_LOG_DIR = "__driver_logs__"
+ val DRIVER_LOG_FILE = "driver.log"
+ val DRIVER_LOG_FILE_SUFFIX = "_" + DRIVER_LOG_FILE
+ val APPENDER_NAME = "_DriverLogAppender"
+
+ def apply(conf: SparkConf): Option[DriverLogger] = {
+ if (conf.get(DRIVER_LOG_PERSISTTODFS) && Utils.isClientMode(conf)) {
+ if (conf.contains(DRIVER_LOG_DFS_DIR)) {
+ try {
+ Some(new DriverLogger(conf))
+ } catch {
+ case e: Exception =>
+ logError("Could not add driver logger", e)
+ None
+ }
+ } else {
+ logWarning(s"Driver logs are not persisted because" +
+ s" ${DRIVER_LOG_DFS_DIR.key} is not configured")
+ None
+ }
+ } else {
+ None
+ }
+ }
+}
diff --git a/core/src/main/scala/org/apache/spark/util/random/RandomSampler.scala b/core/src/main/scala/org/apache/spark/util/random/RandomSampler.scala
index ea99a7e5b4847..70554f1d03067 100644
--- a/core/src/main/scala/org/apache/spark/util/random/RandomSampler.scala
+++ b/core/src/main/scala/org/apache/spark/util/random/RandomSampler.scala
@@ -49,7 +49,7 @@ trait RandomSampler[T, U] extends Pseudorandom with Cloneable with Serializable
/** return a copy of the RandomSampler object */
override def clone: RandomSampler[T, U] =
- throw new NotImplementedError("clone() is not implemented.")
+ throw new UnsupportedOperationException("clone() is not implemented.")
}
private[spark]
diff --git a/core/src/test/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriterSuite.java b/core/src/test/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriterSuite.java
index a07d0e84ea854..30ad3f5575545 100644
--- a/core/src/test/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriterSuite.java
+++ b/core/src/test/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriterSuite.java
@@ -162,7 +162,8 @@ private UnsafeShuffleWriter createWriter(
new SerializedShuffleHandle<>(0, 1, shuffleDep),
0, // map id
taskContext,
- conf
+ conf,
+ taskContext.taskMetrics().shuffleWriteMetrics()
);
}
@@ -521,7 +522,8 @@ public void testPeakMemoryUsed() throws Exception {
new SerializedShuffleHandle<>(0, 1, shuffleDep),
0, // map id
taskContext,
- conf);
+ conf,
+ taskContext.taskMetrics().shuffleWriteMetrics());
// Peak memory should be monotonically increasing. More specifically, every time
// we allocate a new page it should increase by exactly the size of the page.
diff --git a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
index 411cd5cb57331..d1b29d90ad913 100644
--- a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
+++ b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
@@ -38,6 +38,7 @@
import org.apache.spark.executor.TaskMetrics;
import org.apache.spark.internal.config.package$;
import org.apache.spark.memory.TestMemoryManager;
+import org.apache.spark.memory.SparkOutOfMemoryError;
import org.apache.spark.memory.TaskMemoryManager;
import org.apache.spark.serializer.JavaSerializer;
import org.apache.spark.serializer.SerializerInstance;
@@ -534,10 +535,10 @@ public void testOOMDuringSpill() throws Exception {
insertNumber(sorter, 1024);
fail("expected OutOfMmoryError but it seems operation surprisingly succeeded");
}
- // we expect an OutOfMemoryError here, anything else (i.e the original NPE is a failure)
- catch (OutOfMemoryError oom){
+ // we expect an SparkOutOfMemoryError here, anything else (i.e the original NPE is a failure)
+ catch (SparkOutOfMemoryError oom){
String oomStackTrace = Utils.exceptionString(oom);
- assertThat("expected OutOfMemoryError in " +
+ assertThat("expected SparkOutOfMemoryError in " +
"org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset",
oomStackTrace,
Matchers.containsString(
diff --git a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
index 85ffdca436e14..b0d485f0c953f 100644
--- a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
+++ b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
@@ -27,6 +27,7 @@
import org.apache.spark.SparkConf;
import org.apache.spark.memory.TestMemoryConsumer;
import org.apache.spark.memory.TestMemoryManager;
+import org.apache.spark.memory.SparkOutOfMemoryError;
import org.apache.spark.memory.TaskMemoryManager;
import org.apache.spark.unsafe.Platform;
import org.apache.spark.unsafe.memory.MemoryBlock;
@@ -178,8 +179,8 @@ public int compare(
testMemoryManager.markExecutionAsOutOfMemoryOnce();
try {
sorter.reset();
- fail("expected OutOfMmoryError but it seems operation surprisingly succeeded");
- } catch (OutOfMemoryError oom) {
+ fail("expected SparkOutOfMemoryError but it seems operation surprisingly succeeded");
+ } catch (SparkOutOfMemoryError oom) {
// as expected
}
// [SPARK-21907] this failed on NPE at
diff --git a/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json
index 5e9e8230e2745..62e5c123fd3d4 100644
--- a/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json
@@ -1,639 +1,708 @@
{
- "status": "COMPLETE",
- "stageId": 0,
- "attemptId": 0,
- "numTasks": 10,
- "numActiveTasks": 0,
- "numCompleteTasks": 10,
- "numFailedTasks": 2,
- "numKilledTasks": 0,
- "numCompletedIndices": 10,
- "executorRunTime": 761,
- "executorCpuTime": 269916000,
- "submissionTime": "2018-01-09T10:21:18.152GMT",
- "firstTaskLaunchedTime": "2018-01-09T10:21:18.347GMT",
- "completionTime": "2018-01-09T10:21:19.062GMT",
- "inputBytes": 0,
- "inputRecords": 0,
- "outputBytes": 0,
- "outputRecords": 0,
- "shuffleReadBytes": 0,
- "shuffleReadRecords": 0,
- "shuffleWriteBytes": 460,
- "shuffleWriteRecords": 10,
- "memoryBytesSpilled": 0,
- "diskBytesSpilled": 0,
- "name": "map at :26",
- "details": "org.apache.spark.rdd.RDD.map(RDD.scala:370)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:34)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line17.$read$$iw$$iw$$iw$$iw$$iw.(:38)\n$line17.$read$$iw$$iw$$iw$$iw.(:40)\n$line17.$read$$iw$$iw$$iw.(:42)\n$line17.$read$$iw$$iw.(:44)\n$line17.$read$$iw.(:46)\n$line17.$read.(:48)\n$line17.$read$.(:52)\n$line17.$read$.()\n$line17.$eval$.$print$lzycompute(:7)\n$line17.$eval$.$print(:6)\n$line17.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)",
- "schedulingPool": "default",
- "rddIds": [
- 1,
- 0
- ],
- "accumulatorUpdates": [],
- "tasks": {
- "0": {
- "taskId": 0,
- "index": 0,
- "attempt": 0,
- "launchTime": "2018-01-09T10:21:18.347GMT",
- "duration": 562,
- "executorId": "0",
- "host": "172.30.65.138",
- "status": "FAILED",
- "taskLocality": "PROCESS_LOCAL",
- "speculative": false,
- "accumulatorUpdates": [],
- "errorMessage": "java.lang.RuntimeException: Bad executor\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(:27)\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
- "taskMetrics": {
- "executorDeserializeTime": 0,
- "executorDeserializeCpuTime": 0,
- "executorRunTime": 460,
- "executorCpuTime": 0,
- "resultSize": 0,
- "jvmGcTime": 14,
- "resultSerializationTime": 0,
- "memoryBytesSpilled": 0,
- "diskBytesSpilled": 0,
- "peakExecutionMemory": 0,
- "inputMetrics": {
- "bytesRead": 0,
- "recordsRead": 0
- },
- "outputMetrics": {
- "bytesWritten": 0,
- "recordsWritten": 0
- },
- "shuffleReadMetrics": {
- "remoteBlocksFetched": 0,
- "localBlocksFetched": 0,
- "fetchWaitTime": 0,
- "remoteBytesRead": 0,
- "remoteBytesReadToDisk": 0,
- "localBytesRead": 0,
- "recordsRead": 0
- },
- "shuffleWriteMetrics": {
- "bytesWritten": 0,
- "writeTime": 3873006,
- "recordsWritten": 0
+ "status" : "COMPLETE",
+ "stageId" : 0,
+ "attemptId" : 0,
+ "numTasks" : 10,
+ "numActiveTasks" : 0,
+ "numCompleteTasks" : 10,
+ "numFailedTasks" : 2,
+ "numKilledTasks" : 0,
+ "numCompletedIndices" : 10,
+ "executorRunTime" : 761,
+ "executorCpuTime" : 269916000,
+ "submissionTime" : "2018-01-09T10:21:18.152GMT",
+ "firstTaskLaunchedTime" : "2018-01-09T10:21:18.347GMT",
+ "completionTime" : "2018-01-09T10:21:19.062GMT",
+ "inputBytes" : 0,
+ "inputRecords" : 0,
+ "outputBytes" : 0,
+ "outputRecords" : 0,
+ "shuffleReadBytes" : 0,
+ "shuffleReadRecords" : 0,
+ "shuffleWriteBytes" : 460,
+ "shuffleWriteRecords" : 10,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "name" : "map at :26",
+ "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:370)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:34)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line17.$read$$iw$$iw$$iw$$iw$$iw.(:38)\n$line17.$read$$iw$$iw$$iw$$iw.(:40)\n$line17.$read$$iw$$iw$$iw.(:42)\n$line17.$read$$iw$$iw.(:44)\n$line17.$read$$iw.(:46)\n$line17.$read.(:48)\n$line17.$read$.(:52)\n$line17.$read$.()\n$line17.$eval$.$print$lzycompute(:7)\n$line17.$eval$.$print(:6)\n$line17.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)",
+ "schedulingPool" : "default",
+ "rddIds" : [ 1, 0 ],
+ "accumulatorUpdates" : [ ],
+ "tasks" : {
+ "0" : {
+ "taskId" : 0,
+ "index" : 0,
+ "attempt" : 0,
+ "launchTime" : "2018-01-09T10:21:18.347GMT",
+ "duration" : 562,
+ "executorId" : "0",
+ "host" : "172.30.65.138",
+ "status" : "FAILED",
+ "taskLocality" : "PROCESS_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "errorMessage" : "java.lang.RuntimeException: Bad executor\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(:27)\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
+ "taskMetrics" : {
+ "executorDeserializeTime" : 0,
+ "executorDeserializeCpuTime" : 0,
+ "executorRunTime" : 460,
+ "executorCpuTime" : 0,
+ "resultSize" : 0,
+ "jvmGcTime" : 14,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 3873006,
+ "recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stdout",
+ "stderr" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stderr"
+ },
+ "schedulerDelay" : 102,
+ "gettingResultTime" : 0
},
- "5": {
- "taskId": 5,
- "index": 3,
- "attempt": 0,
- "launchTime": "2018-01-09T10:21:18.958GMT",
- "duration": 22,
- "executorId": "1",
- "host": "172.30.65.138",
- "status": "SUCCESS",
- "taskLocality": "PROCESS_LOCAL",
- "speculative": false,
- "accumulatorUpdates": [],
- "taskMetrics": {
- "executorDeserializeTime": 3,
- "executorDeserializeCpuTime": 2586000,
- "executorRunTime": 9,
- "executorCpuTime": 9635000,
- "resultSize": 1029,
- "jvmGcTime": 0,
- "resultSerializationTime": 0,
- "memoryBytesSpilled": 0,
- "diskBytesSpilled": 0,
- "peakExecutionMemory": 0,
- "inputMetrics": {
- "bytesRead": 0,
- "recordsRead": 0
- },
- "outputMetrics": {
- "bytesWritten": 0,
- "recordsWritten": 0
- },
- "shuffleReadMetrics": {
- "remoteBlocksFetched": 0,
- "localBlocksFetched": 0,
- "fetchWaitTime": 0,
- "remoteBytesRead": 0,
- "remoteBytesReadToDisk": 0,
- "localBytesRead": 0,
- "recordsRead": 0
- },
- "shuffleWriteMetrics": {
- "bytesWritten": 46,
- "writeTime": 262919,
- "recordsWritten": 1
+ "5" : {
+ "taskId" : 5,
+ "index" : 3,
+ "attempt" : 0,
+ "launchTime" : "2018-01-09T10:21:18.958GMT",
+ "duration" : 22,
+ "executorId" : "1",
+ "host" : "172.30.65.138",
+ "status" : "SUCCESS",
+ "taskLocality" : "PROCESS_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 3,
+ "executorDeserializeCpuTime" : 2586000,
+ "executorRunTime" : 9,
+ "executorCpuTime" : 9635000,
+ "resultSize" : 1029,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 46,
+ "writeTime" : 262919,
+ "recordsWritten" : 1
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+ "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+ },
+ "schedulerDelay" : 10,
+ "gettingResultTime" : 0
},
- "10": {
- "taskId": 10,
- "index": 8,
- "attempt": 0,
- "launchTime": "2018-01-09T10:21:19.034GMT",
- "duration": 12,
- "executorId": "1",
- "host": "172.30.65.138",
- "status": "SUCCESS",
- "taskLocality": "PROCESS_LOCAL",
- "speculative": false,
- "accumulatorUpdates": [],
- "taskMetrics": {
- "executorDeserializeTime": 2,
- "executorDeserializeCpuTime": 1803000,
- "executorRunTime": 6,
- "executorCpuTime": 6157000,
- "resultSize": 1029,
- "jvmGcTime": 0,
- "resultSerializationTime": 0,
- "memoryBytesSpilled": 0,
- "diskBytesSpilled": 0,
- "peakExecutionMemory": 0,
- "inputMetrics": {
- "bytesRead": 0,
- "recordsRead": 0
- },
- "outputMetrics": {
- "bytesWritten": 0,
- "recordsWritten": 0
- },
- "shuffleReadMetrics": {
- "remoteBlocksFetched": 0,
- "localBlocksFetched": 0,
- "fetchWaitTime": 0,
- "remoteBytesRead": 0,
- "remoteBytesReadToDisk": 0,
- "localBytesRead": 0,
- "recordsRead": 0
- },
- "shuffleWriteMetrics": {
- "bytesWritten": 46,
- "writeTime": 243647,
- "recordsWritten": 1
+ "10" : {
+ "taskId" : 10,
+ "index" : 8,
+ "attempt" : 0,
+ "launchTime" : "2018-01-09T10:21:19.034GMT",
+ "duration" : 12,
+ "executorId" : "1",
+ "host" : "172.30.65.138",
+ "status" : "SUCCESS",
+ "taskLocality" : "PROCESS_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 2,
+ "executorDeserializeCpuTime" : 1803000,
+ "executorRunTime" : 6,
+ "executorCpuTime" : 6157000,
+ "resultSize" : 1029,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 46,
+ "writeTime" : 243647,
+ "recordsWritten" : 1
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+ "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+ },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
},
- "1": {
- "taskId": 1,
- "index": 1,
- "attempt": 0,
- "launchTime": "2018-01-09T10:21:18.364GMT",
- "duration": 565,
- "executorId": "1",
- "host": "172.30.65.138",
- "status": "SUCCESS",
- "taskLocality": "PROCESS_LOCAL",
- "speculative": false,
- "accumulatorUpdates": [],
- "taskMetrics": {
- "executorDeserializeTime": 301,
- "executorDeserializeCpuTime": 200029000,
- "executorRunTime": 212,
- "executorCpuTime": 198479000,
- "resultSize": 1115,
- "jvmGcTime": 13,
- "resultSerializationTime": 1,
- "memoryBytesSpilled": 0,
- "diskBytesSpilled": 0,
- "peakExecutionMemory": 0,
- "inputMetrics": {
- "bytesRead": 0,
- "recordsRead": 0
- },
- "outputMetrics": {
- "bytesWritten": 0,
- "recordsWritten": 0
- },
- "shuffleReadMetrics": {
- "remoteBlocksFetched": 0,
- "localBlocksFetched": 0,
- "fetchWaitTime": 0,
- "remoteBytesRead": 0,
- "remoteBytesReadToDisk": 0,
- "localBytesRead": 0,
- "recordsRead": 0
- },
- "shuffleWriteMetrics": {
- "bytesWritten": 46,
- "writeTime": 2409488,
- "recordsWritten": 1
+ "1" : {
+ "taskId" : 1,
+ "index" : 1,
+ "attempt" : 0,
+ "launchTime" : "2018-01-09T10:21:18.364GMT",
+ "duration" : 565,
+ "executorId" : "1",
+ "host" : "172.30.65.138",
+ "status" : "SUCCESS",
+ "taskLocality" : "PROCESS_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 301,
+ "executorDeserializeCpuTime" : 200029000,
+ "executorRunTime" : 212,
+ "executorCpuTime" : 198479000,
+ "resultSize" : 1115,
+ "jvmGcTime" : 13,
+ "resultSerializationTime" : 1,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 46,
+ "writeTime" : 2409488,
+ "recordsWritten" : 1
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+ "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+ },
+ "schedulerDelay" : 51,
+ "gettingResultTime" : 0
},
- "6": {
- "taskId": 6,
- "index": 4,
- "attempt": 0,
- "launchTime": "2018-01-09T10:21:18.980GMT",
- "duration": 16,
- "executorId": "1",
- "host": "172.30.65.138",
- "status": "SUCCESS",
- "taskLocality": "PROCESS_LOCAL",
- "speculative": false,
- "accumulatorUpdates": [],
- "taskMetrics": {
- "executorDeserializeTime": 3,
- "executorDeserializeCpuTime": 2610000,
- "executorRunTime": 10,
- "executorCpuTime": 9622000,
- "resultSize": 1029,
- "jvmGcTime": 0,
- "resultSerializationTime": 0,
- "memoryBytesSpilled": 0,
- "diskBytesSpilled": 0,
- "peakExecutionMemory": 0,
- "inputMetrics": {
- "bytesRead": 0,
- "recordsRead": 0
- },
- "outputMetrics": {
- "bytesWritten": 0,
- "recordsWritten": 0
- },
- "shuffleReadMetrics": {
- "remoteBlocksFetched": 0,
- "localBlocksFetched": 0,
- "fetchWaitTime": 0,
- "remoteBytesRead": 0,
- "remoteBytesReadToDisk": 0,
- "localBytesRead": 0,
- "recordsRead": 0
- },
- "shuffleWriteMetrics": {
- "bytesWritten": 46,
- "writeTime": 385110,
- "recordsWritten": 1
+ "6" : {
+ "taskId" : 6,
+ "index" : 4,
+ "attempt" : 0,
+ "launchTime" : "2018-01-09T10:21:18.980GMT",
+ "duration" : 16,
+ "executorId" : "1",
+ "host" : "172.30.65.138",
+ "status" : "SUCCESS",
+ "taskLocality" : "PROCESS_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 3,
+ "executorDeserializeCpuTime" : 2610000,
+ "executorRunTime" : 10,
+ "executorCpuTime" : 9622000,
+ "resultSize" : 1029,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 46,
+ "writeTime" : 385110,
+ "recordsWritten" : 1
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+ "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+ },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
},
- "9": {
- "taskId": 9,
- "index": 7,
- "attempt": 0,
- "launchTime": "2018-01-09T10:21:19.022GMT",
- "duration": 12,
- "executorId": "1",
- "host": "172.30.65.138",
- "status": "SUCCESS",
- "taskLocality": "PROCESS_LOCAL",
- "speculative": false,
- "accumulatorUpdates": [],
- "taskMetrics": {
- "executorDeserializeTime": 2,
- "executorDeserializeCpuTime": 1981000,
- "executorRunTime": 7,
- "executorCpuTime": 6335000,
- "resultSize": 1029,
- "jvmGcTime": 0,
- "resultSerializationTime": 0,
- "memoryBytesSpilled": 0,
- "diskBytesSpilled": 0,
- "peakExecutionMemory": 0,
- "inputMetrics": {
- "bytesRead": 0,
- "recordsRead": 0
- },
- "outputMetrics": {
- "bytesWritten": 0,
- "recordsWritten": 0
- },
- "shuffleReadMetrics": {
- "remoteBlocksFetched": 0,
- "localBlocksFetched": 0,
- "fetchWaitTime": 0,
- "remoteBytesRead": 0,
- "remoteBytesReadToDisk": 0,
- "localBytesRead": 0,
- "recordsRead": 0
- },
- "shuffleWriteMetrics": {
- "bytesWritten": 46,
- "writeTime": 259354,
- "recordsWritten": 1
+ "9" : {
+ "taskId" : 9,
+ "index" : 7,
+ "attempt" : 0,
+ "launchTime" : "2018-01-09T10:21:19.022GMT",
+ "duration" : 12,
+ "executorId" : "1",
+ "host" : "172.30.65.138",
+ "status" : "SUCCESS",
+ "taskLocality" : "PROCESS_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 2,
+ "executorDeserializeCpuTime" : 1981000,
+ "executorRunTime" : 7,
+ "executorCpuTime" : 6335000,
+ "resultSize" : 1029,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 46,
+ "writeTime" : 259354,
+ "recordsWritten" : 1
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+ "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+ },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
},
- "2": {
- "taskId": 2,
- "index": 2,
- "attempt": 0,
- "launchTime": "2018-01-09T10:21:18.899GMT",
- "duration": 27,
- "executorId": "0",
- "host": "172.30.65.138",
- "status": "FAILED",
- "taskLocality": "PROCESS_LOCAL",
- "speculative": false,
- "accumulatorUpdates": [],
- "errorMessage": "java.lang.RuntimeException: Bad executor\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(:27)\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
- "taskMetrics": {
- "executorDeserializeTime": 0,
- "executorDeserializeCpuTime": 0,
- "executorRunTime": 16,
- "executorCpuTime": 0,
- "resultSize": 0,
- "jvmGcTime": 0,
- "resultSerializationTime": 0,
- "memoryBytesSpilled": 0,
- "diskBytesSpilled": 0,
- "peakExecutionMemory": 0,
- "inputMetrics": {
- "bytesRead": 0,
- "recordsRead": 0
- },
- "outputMetrics": {
- "bytesWritten": 0,
- "recordsWritten": 0
- },
- "shuffleReadMetrics": {
- "remoteBlocksFetched": 0,
- "localBlocksFetched": 0,
- "fetchWaitTime": 0,
- "remoteBytesRead": 0,
- "remoteBytesReadToDisk": 0,
- "localBytesRead": 0,
- "recordsRead": 0
- },
- "shuffleWriteMetrics": {
- "bytesWritten": 0,
- "writeTime": 126128,
- "recordsWritten": 0
+ "2" : {
+ "taskId" : 2,
+ "index" : 2,
+ "attempt" : 0,
+ "launchTime" : "2018-01-09T10:21:18.899GMT",
+ "duration" : 27,
+ "executorId" : "0",
+ "host" : "172.30.65.138",
+ "status" : "FAILED",
+ "taskLocality" : "PROCESS_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "errorMessage" : "java.lang.RuntimeException: Bad executor\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(:27)\n\tat $line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$2.apply(:26)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:409)\n\tat org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:193)\n\tat org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:109)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
+ "taskMetrics" : {
+ "executorDeserializeTime" : 0,
+ "executorDeserializeCpuTime" : 0,
+ "executorRunTime" : 16,
+ "executorCpuTime" : 0,
+ "resultSize" : 0,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 126128,
+ "recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stdout",
+ "stderr" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stderr"
+ },
+ "schedulerDelay" : 11,
+ "gettingResultTime" : 0
},
- "7": {
- "taskId": 7,
- "index": 5,
- "attempt": 0,
- "launchTime": "2018-01-09T10:21:18.996GMT",
- "duration": 15,
- "executorId": "1",
- "host": "172.30.65.138",
- "status": "SUCCESS",
- "taskLocality": "PROCESS_LOCAL",
- "speculative": false,
- "accumulatorUpdates": [],
- "taskMetrics": {
- "executorDeserializeTime": 2,
- "executorDeserializeCpuTime": 2231000,
- "executorRunTime": 9,
- "executorCpuTime": 8407000,
- "resultSize": 1029,
- "jvmGcTime": 0,
- "resultSerializationTime": 0,
- "memoryBytesSpilled": 0,
- "diskBytesSpilled": 0,
- "peakExecutionMemory": 0,
- "inputMetrics": {
- "bytesRead": 0,
- "recordsRead": 0
- },
- "outputMetrics": {
- "bytesWritten": 0,
- "recordsWritten": 0
- },
- "shuffleReadMetrics": {
- "remoteBlocksFetched": 0,
- "localBlocksFetched": 0,
- "fetchWaitTime": 0,
- "remoteBytesRead": 0,
- "remoteBytesReadToDisk": 0,
- "localBytesRead": 0,
- "recordsRead": 0
- },
- "shuffleWriteMetrics": {
- "bytesWritten": 46,
- "writeTime": 205520,
- "recordsWritten": 1
+ "7" : {
+ "taskId" : 7,
+ "index" : 5,
+ "attempt" : 0,
+ "launchTime" : "2018-01-09T10:21:18.996GMT",
+ "duration" : 15,
+ "executorId" : "1",
+ "host" : "172.30.65.138",
+ "status" : "SUCCESS",
+ "taskLocality" : "PROCESS_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 2,
+ "executorDeserializeCpuTime" : 2231000,
+ "executorRunTime" : 9,
+ "executorCpuTime" : 8407000,
+ "resultSize" : 1029,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 46,
+ "writeTime" : 205520,
+ "recordsWritten" : 1
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+ "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+ },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
},
- "3": {
- "taskId": 3,
- "index": 0,
- "attempt": 1,
- "launchTime": "2018-01-09T10:21:18.919GMT",
- "duration": 24,
- "executorId": "1",
- "host": "172.30.65.138",
- "status": "SUCCESS",
- "taskLocality": "PROCESS_LOCAL",
- "speculative": false,
- "accumulatorUpdates": [],
- "taskMetrics": {
- "executorDeserializeTime": 8,
- "executorDeserializeCpuTime": 8878000,
- "executorRunTime": 10,
- "executorCpuTime": 9364000,
- "resultSize": 1029,
- "jvmGcTime": 0,
- "resultSerializationTime": 0,
- "memoryBytesSpilled": 0,
- "diskBytesSpilled": 0,
- "peakExecutionMemory": 0,
- "inputMetrics": {
- "bytesRead": 0,
- "recordsRead": 0
- },
- "outputMetrics": {
- "bytesWritten": 0,
- "recordsWritten": 0
- },
- "shuffleReadMetrics": {
- "remoteBlocksFetched": 0,
- "localBlocksFetched": 0,
- "fetchWaitTime": 0,
- "remoteBytesRead": 0,
- "remoteBytesReadToDisk": 0,
- "localBytesRead": 0,
- "recordsRead": 0
- },
- "shuffleWriteMetrics": {
- "bytesWritten": 46,
- "writeTime": 207014,
- "recordsWritten": 1
+ "3" : {
+ "taskId" : 3,
+ "index" : 0,
+ "attempt" : 1,
+ "launchTime" : "2018-01-09T10:21:18.919GMT",
+ "duration" : 24,
+ "executorId" : "1",
+ "host" : "172.30.65.138",
+ "status" : "SUCCESS",
+ "taskLocality" : "PROCESS_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 8,
+ "executorDeserializeCpuTime" : 8878000,
+ "executorRunTime" : 10,
+ "executorCpuTime" : 9364000,
+ "resultSize" : 1029,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 46,
+ "writeTime" : 207014,
+ "recordsWritten" : 1
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+ "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+ },
+ "schedulerDelay" : 6,
+ "gettingResultTime" : 0
},
- "11": {
- "taskId": 11,
- "index": 9,
- "attempt": 0,
- "launchTime": "2018-01-09T10:21:19.045GMT",
- "duration": 15,
- "executorId": "1",
- "host": "172.30.65.138",
- "status": "SUCCESS",
- "taskLocality": "PROCESS_LOCAL",
- "speculative": false,
- "accumulatorUpdates": [],
- "taskMetrics": {
- "executorDeserializeTime": 3,
- "executorDeserializeCpuTime": 2017000,
- "executorRunTime": 6,
- "executorCpuTime": 6676000,
- "resultSize": 1029,
- "jvmGcTime": 0,
- "resultSerializationTime": 0,
- "memoryBytesSpilled": 0,
- "diskBytesSpilled": 0,
- "peakExecutionMemory": 0,
- "inputMetrics": {
- "bytesRead": 0,
- "recordsRead": 0
- },
- "outputMetrics": {
- "bytesWritten": 0,
- "recordsWritten": 0
- },
- "shuffleReadMetrics": {
- "remoteBlocksFetched": 0,
- "localBlocksFetched": 0,
- "fetchWaitTime": 0,
- "remoteBytesRead": 0,
- "remoteBytesReadToDisk": 0,
- "localBytesRead": 0,
- "recordsRead": 0
- },
- "shuffleWriteMetrics": {
- "bytesWritten": 46,
- "writeTime": 233652,
- "recordsWritten": 1
+ "11" : {
+ "taskId" : 11,
+ "index" : 9,
+ "attempt" : 0,
+ "launchTime" : "2018-01-09T10:21:19.045GMT",
+ "duration" : 15,
+ "executorId" : "1",
+ "host" : "172.30.65.138",
+ "status" : "SUCCESS",
+ "taskLocality" : "PROCESS_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 3,
+ "executorDeserializeCpuTime" : 2017000,
+ "executorRunTime" : 6,
+ "executorCpuTime" : 6676000,
+ "resultSize" : 1029,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 46,
+ "writeTime" : 233652,
+ "recordsWritten" : 1
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+ "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+ },
+ "schedulerDelay" : 6,
+ "gettingResultTime" : 0
},
- "8": {
- "taskId": 8,
- "index": 6,
- "attempt": 0,
- "launchTime": "2018-01-09T10:21:19.011GMT",
- "duration": 11,
- "executorId": "1",
- "host": "172.30.65.138",
- "status": "SUCCESS",
- "taskLocality": "PROCESS_LOCAL",
- "speculative": false,
- "accumulatorUpdates": [],
- "taskMetrics": {
- "executorDeserializeTime": 1,
- "executorDeserializeCpuTime": 1554000,
- "executorRunTime": 7,
- "executorCpuTime": 6034000,
- "resultSize": 1029,
- "jvmGcTime": 0,
- "resultSerializationTime": 0,
- "memoryBytesSpilled": 0,
- "diskBytesSpilled": 0,
- "peakExecutionMemory": 0,
- "inputMetrics": {
- "bytesRead": 0,
- "recordsRead": 0
- },
- "outputMetrics": {
- "bytesWritten": 0,
- "recordsWritten": 0
- },
- "shuffleReadMetrics": {
- "remoteBlocksFetched": 0,
- "localBlocksFetched": 0,
- "fetchWaitTime": 0,
- "remoteBytesRead": 0,
- "remoteBytesReadToDisk": 0,
- "localBytesRead": 0,
- "recordsRead": 0
- },
- "shuffleWriteMetrics": {
- "bytesWritten": 46,
- "writeTime": 213296,
- "recordsWritten": 1
+ "8" : {
+ "taskId" : 8,
+ "index" : 6,
+ "attempt" : 0,
+ "launchTime" : "2018-01-09T10:21:19.011GMT",
+ "duration" : 11,
+ "executorId" : "1",
+ "host" : "172.30.65.138",
+ "status" : "SUCCESS",
+ "taskLocality" : "PROCESS_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 1,
+ "executorDeserializeCpuTime" : 1554000,
+ "executorRunTime" : 7,
+ "executorCpuTime" : 6034000,
+ "resultSize" : 1029,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 46,
+ "writeTime" : 213296,
+ "recordsWritten" : 1
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+ "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+ },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
},
- "4": {
- "taskId": 4,
- "index": 2,
- "attempt": 1,
- "launchTime": "2018-01-09T10:21:18.943GMT",
- "duration": 16,
- "executorId": "1",
- "host": "172.30.65.138",
- "status": "SUCCESS",
- "taskLocality": "PROCESS_LOCAL",
- "speculative": false,
- "accumulatorUpdates": [],
- "taskMetrics": {
- "executorDeserializeTime": 2,
- "executorDeserializeCpuTime": 2211000,
- "executorRunTime": 9,
- "executorCpuTime": 9207000,
- "resultSize": 1029,
- "jvmGcTime": 0,
- "resultSerializationTime": 0,
- "memoryBytesSpilled": 0,
- "diskBytesSpilled": 0,
- "peakExecutionMemory": 0,
- "inputMetrics": {
- "bytesRead": 0,
- "recordsRead": 0
- },
- "outputMetrics": {
- "bytesWritten": 0,
- "recordsWritten": 0
- },
- "shuffleReadMetrics": {
- "remoteBlocksFetched": 0,
- "localBlocksFetched": 0,
- "fetchWaitTime": 0,
- "remoteBytesRead": 0,
- "remoteBytesReadToDisk": 0,
- "localBytesRead": 0,
- "recordsRead": 0
- },
- "shuffleWriteMetrics": {
- "bytesWritten": 46,
- "writeTime": 292381,
- "recordsWritten": 1
+ "4" : {
+ "taskId" : 4,
+ "index" : 2,
+ "attempt" : 1,
+ "launchTime" : "2018-01-09T10:21:18.943GMT",
+ "duration" : 16,
+ "executorId" : "1",
+ "host" : "172.30.65.138",
+ "status" : "SUCCESS",
+ "taskLocality" : "PROCESS_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 2,
+ "executorDeserializeCpuTime" : 2211000,
+ "executorRunTime" : 9,
+ "executorCpuTime" : 9207000,
+ "resultSize" : 1029,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 46,
+ "writeTime" : 292381,
+ "recordsWritten" : 1
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout",
+ "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr"
+ },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
}
},
- "executorSummary": {
- "0": {
- "taskTime": 589,
- "failedTasks": 2,
- "succeededTasks": 0,
- "killedTasks": 0,
- "inputBytes": 0,
- "inputRecords": 0,
- "outputBytes": 0,
- "outputRecords": 0,
- "shuffleRead": 0,
- "shuffleReadRecords": 0,
- "shuffleWrite": 0,
- "shuffleWriteRecords": 0,
- "memoryBytesSpilled": 0,
- "diskBytesSpilled": 0,
- "isBlacklistedForStage": true
+ "executorSummary" : {
+ "0" : {
+ "taskTime" : 589,
+ "failedTasks" : 2,
+ "succeededTasks" : 0,
+ "killedTasks" : 0,
+ "inputBytes" : 0,
+ "inputRecords" : 0,
+ "outputBytes" : 0,
+ "outputRecords" : 0,
+ "shuffleRead" : 0,
+ "shuffleReadRecords" : 0,
+ "shuffleWrite" : 0,
+ "shuffleWriteRecords" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "isBlacklistedForStage" : true
},
- "1": {
- "taskTime": 708,
- "failedTasks": 0,
- "succeededTasks": 10,
- "killedTasks": 0,
- "inputBytes": 0,
- "inputRecords": 0,
- "outputBytes": 0,
- "outputRecords": 0,
- "shuffleRead": 0,
- "shuffleReadRecords": 0,
- "shuffleWrite": 460,
- "shuffleWriteRecords": 10,
- "memoryBytesSpilled": 0,
- "diskBytesSpilled": 0,
- "isBlacklistedForStage": false
+ "1" : {
+ "taskTime" : 708,
+ "failedTasks" : 0,
+ "succeededTasks" : 10,
+ "killedTasks" : 0,
+ "inputBytes" : 0,
+ "inputRecords" : 0,
+ "outputBytes" : 0,
+ "outputRecords" : 0,
+ "shuffleRead" : 0,
+ "shuffleReadRecords" : 0,
+ "shuffleWrite" : 460,
+ "shuffleWriteRecords" : 10,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "isBlacklistedForStage" : false
}
},
- "killedTasksSummary": {}
+ "killedTasksSummary" : { }
}
diff --git a/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json
index acd4cc53de6cd..6e46c881b2a21 100644
--- a/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json
@@ -74,7 +74,13 @@
"writeTime" : 3662221,
"recordsWritten" : 3
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096",
+ "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096"
+ },
+ "schedulerDelay" : 68,
+ "gettingResultTime" : 0
},
"5" : {
"taskId" : 5,
@@ -122,7 +128,13 @@
"writeTime" : 191901,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stdout?start=-4096",
+ "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stderr?start=-4096"
+ },
+ "schedulerDelay" : 46,
+ "gettingResultTime" : 0
},
"10" : {
"taskId" : 10,
@@ -169,7 +181,13 @@
"writeTime" : 301705,
"recordsWritten" : 3
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096",
+ "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096"
+ },
+ "schedulerDelay" : 50,
+ "gettingResultTime" : 0
},
"1" : {
"taskId" : 1,
@@ -217,7 +235,13 @@
"writeTime" : 3075188,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stdout?start=-4096",
+ "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stderr?start=-4096"
+ },
+ "schedulerDelay" : 174,
+ "gettingResultTime" : 0
},
"6" : {
"taskId" : 6,
@@ -265,7 +289,13 @@
"writeTime" : 183718,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stdout?start=-4096",
+ "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stderr?start=-4096"
+ },
+ "schedulerDelay" : 16,
+ "gettingResultTime" : 0
},
"9" : {
"taskId" : 9,
@@ -312,7 +342,13 @@
"writeTime" : 366050,
"recordsWritten" : 3
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096",
+ "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096"
+ },
+ "schedulerDelay" : 42,
+ "gettingResultTime" : 0
},
"13" : {
"taskId" : 13,
@@ -359,7 +395,13 @@
"writeTime" : 369513,
"recordsWritten" : 3
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096",
+ "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096"
+ },
+ "schedulerDelay" : 26,
+ "gettingResultTime" : 0
},
"2" : {
"taskId" : 2,
@@ -406,7 +448,13 @@
"writeTime" : 3322956,
"recordsWritten" : 3
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000004/attilapiros/stdout?start=-4096",
+ "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000004/attilapiros/stderr?start=-4096"
+ },
+ "schedulerDelay" : 74,
+ "gettingResultTime" : 0
},
"12" : {
"taskId" : 12,
@@ -453,7 +501,13 @@
"writeTime" : 319101,
"recordsWritten" : 3
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096",
+ "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096"
+ },
+ "schedulerDelay" : 22,
+ "gettingResultTime" : 0
},
"7" : {
"taskId" : 7,
@@ -500,7 +554,13 @@
"writeTime" : 377601,
"recordsWritten" : 3
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096",
+ "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096"
+ },
+ "schedulerDelay" : 20,
+ "gettingResultTime" : 0
},
"3" : {
"taskId" : 3,
@@ -547,7 +607,13 @@
"writeTime" : 3587839,
"recordsWritten" : 3
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096",
+ "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096"
+ },
+ "schedulerDelay" : 63,
+ "gettingResultTime" : 0
},
"11" : {
"taskId" : 11,
@@ -594,7 +660,13 @@
"writeTime" : 323898,
"recordsWritten" : 3
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096",
+ "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096"
+ },
+ "schedulerDelay" : 12,
+ "gettingResultTime" : 0
},
"8" : {
"taskId" : 8,
@@ -641,7 +713,13 @@
"writeTime" : 311940,
"recordsWritten" : 3
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096",
+ "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096"
+ },
+ "schedulerDelay" : 84,
+ "gettingResultTime" : 0
},
"4" : {
"taskId" : 4,
@@ -689,7 +767,13 @@
"writeTime" : 16858066,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : {
+ "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stdout?start=-4096",
+ "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stderr?start=-4096"
+ },
+ "schedulerDelay" : 338,
+ "gettingResultTime" : 0
}
},
"executorSummary" : {
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
index 03f886afa5413..aa9471301fe3e 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
@@ -74,7 +74,10 @@
"writeTime" : 76000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 19,
+ "gettingResultTime" : 0
},
"14" : {
"taskId" : 14,
@@ -121,7 +124,10 @@
"writeTime" : 88000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 13,
+ "gettingResultTime" : 0
},
"9" : {
"taskId" : 9,
@@ -168,7 +174,10 @@
"writeTime" : 98000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 17,
+ "gettingResultTime" : 0
},
"13" : {
"taskId" : 13,
@@ -215,7 +224,10 @@
"writeTime" : 73000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 14,
+ "gettingResultTime" : 0
},
"12" : {
"taskId" : 12,
@@ -262,7 +274,10 @@
"writeTime" : 101000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 17,
+ "gettingResultTime" : 0
},
"11" : {
"taskId" : 11,
@@ -309,7 +324,10 @@
"writeTime" : 83000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 17,
+ "gettingResultTime" : 0
},
"8" : {
"taskId" : 8,
@@ -356,7 +374,10 @@
"writeTime" : 94000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 16,
+ "gettingResultTime" : 0
},
"15" : {
"taskId" : 15,
@@ -403,7 +424,10 @@
"writeTime" : 79000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 13,
+ "gettingResultTime" : 0
}
},
"executorSummary" : {
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
index 947c89906955d..584803b5e8631 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
@@ -74,7 +74,10 @@
"writeTime" : 76000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 19,
+ "gettingResultTime" : 0
},
"14" : {
"taskId" : 14,
@@ -121,7 +124,10 @@
"writeTime" : 88000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 13,
+ "gettingResultTime" : 0
},
"9" : {
"taskId" : 9,
@@ -168,7 +174,10 @@
"writeTime" : 98000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 17,
+ "gettingResultTime" : 0
},
"13" : {
"taskId" : 13,
@@ -215,7 +224,10 @@
"writeTime" : 73000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 14,
+ "gettingResultTime" : 0
},
"12" : {
"taskId" : 12,
@@ -262,7 +274,10 @@
"writeTime" : 101000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 17,
+ "gettingResultTime" : 0
},
"11" : {
"taskId" : 11,
@@ -309,7 +324,10 @@
"writeTime" : 83000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 17,
+ "gettingResultTime" : 0
},
"8" : {
"taskId" : 8,
@@ -356,7 +374,10 @@
"writeTime" : 94000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 16,
+ "gettingResultTime" : 0
},
"15" : {
"taskId" : 15,
@@ -403,7 +424,10 @@
"writeTime" : 79000,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 13,
+ "gettingResultTime" : 0
}
},
"executorSummary" : {
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
index a15ee23523365..f859ab6fff240 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
@@ -43,7 +43,10 @@
"writeTime" : 3842811,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 53,
+ "gettingResultTime" : 0
}, {
"taskId" : 1,
"index" : 1,
@@ -89,7 +92,10 @@
"writeTime" : 3934399,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 40,
+ "gettingResultTime" : 0
}, {
"taskId" : 2,
"index" : 2,
@@ -135,7 +141,10 @@
"writeTime" : 89885,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 37,
+ "gettingResultTime" : 0
}, {
"taskId" : 3,
"index" : 3,
@@ -181,7 +190,10 @@
"writeTime" : 1311694,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 41,
+ "gettingResultTime" : 0
}, {
"taskId" : 4,
"index" : 4,
@@ -227,7 +239,10 @@
"writeTime" : 83022,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 38,
+ "gettingResultTime" : 0
}, {
"taskId" : 5,
"index" : 5,
@@ -273,7 +288,10 @@
"writeTime" : 3675510,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 33,
+ "gettingResultTime" : 0
}, {
"taskId" : 6,
"index" : 6,
@@ -319,7 +337,10 @@
"writeTime" : 4016617,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 38,
+ "gettingResultTime" : 0
}, {
"taskId" : 7,
"index" : 7,
@@ -365,7 +386,10 @@
"writeTime" : 2579051,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 43,
+ "gettingResultTime" : 0
}, {
"taskId" : 8,
"index" : 8,
@@ -411,7 +435,10 @@
"writeTime" : 121551,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
}, {
"taskId" : 9,
"index" : 9,
@@ -457,7 +484,10 @@
"writeTime" : 101664,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 8,
+ "gettingResultTime" : 0
}, {
"taskId" : 10,
"index" : 10,
@@ -503,7 +533,10 @@
"writeTime" : 94709,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 18,
+ "gettingResultTime" : 0
}, {
"taskId" : 11,
"index" : 11,
@@ -549,7 +582,10 @@
"writeTime" : 94507,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 8,
+ "gettingResultTime" : 0
}, {
"taskId" : 12,
"index" : 12,
@@ -595,7 +631,10 @@
"writeTime" : 102476,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 7,
+ "gettingResultTime" : 0
}, {
"taskId" : 13,
"index" : 13,
@@ -641,7 +680,10 @@
"writeTime" : 95004,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 53,
+ "gettingResultTime" : 0
}, {
"taskId" : 14,
"index" : 14,
@@ -687,7 +729,10 @@
"writeTime" : 95646,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
}, {
"taskId" : 15,
"index" : 15,
@@ -733,7 +778,10 @@
"writeTime" : 602780,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 16,
"index" : 16,
@@ -779,7 +827,10 @@
"writeTime" : 108320,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 17,
"index" : 17,
@@ -825,7 +876,10 @@
"writeTime" : 99944,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 20,
+ "gettingResultTime" : 0
}, {
"taskId" : 18,
"index" : 18,
@@ -871,7 +925,10 @@
"writeTime" : 100836,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
}, {
"taskId" : 19,
"index" : 19,
@@ -917,5 +974,8 @@
"writeTime" : 95788,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
index f9182b1658334..ea88ca116707a 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
@@ -48,7 +48,10 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 30,
+ "gettingResultTime" : 0
}, {
"taskId" : 1,
"index" : 1,
@@ -99,7 +102,10 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 22,
+ "gettingResultTime" : 0
}, {
"taskId" : 2,
"index" : 2,
@@ -150,7 +156,10 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 18,
+ "gettingResultTime" : 0
}, {
"taskId" : 3,
"index" : 3,
@@ -201,7 +210,10 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 20,
+ "gettingResultTime" : 0
}, {
"taskId" : 4,
"index" : 4,
@@ -252,7 +264,10 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 24,
+ "gettingResultTime" : 0
}, {
"taskId" : 5,
"index" : 5,
@@ -303,7 +318,10 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 23,
+ "gettingResultTime" : 0
}, {
"taskId" : 6,
"index" : 6,
@@ -354,7 +372,10 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 22,
+ "gettingResultTime" : 0
}, {
"taskId" : 7,
"index" : 7,
@@ -405,5 +426,8 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 22,
+ "gettingResultTime" : 0
} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
index 76dd2f710b90f..efd0a45bf01d0 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
@@ -48,7 +48,10 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 30,
+ "gettingResultTime" : 0
}, {
"taskId" : 1,
"index" : 1,
@@ -99,7 +102,10 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 22,
+ "gettingResultTime" : 0
}, {
"taskId" : 2,
"index" : 2,
@@ -150,7 +156,10 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 18,
+ "gettingResultTime" : 0
}, {
"taskId" : 3,
"index" : 3,
@@ -201,7 +210,10 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 20,
+ "gettingResultTime" : 0
}, {
"taskId" : 4,
"index" : 4,
@@ -252,7 +264,10 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 24,
+ "gettingResultTime" : 0
}, {
"taskId" : 5,
"index" : 5,
@@ -303,7 +318,10 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 23,
+ "gettingResultTime" : 0
}, {
"taskId" : 6,
"index" : 6,
@@ -354,7 +372,10 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 22,
+ "gettingResultTime" : 0
}, {
"taskId" : 7,
"index" : 7,
@@ -405,5 +426,8 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 22,
+ "gettingResultTime" : 0
} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
index 6bdc10465d89e..d83528d84972c 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
@@ -43,7 +43,10 @@
"writeTime" : 94709,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 18,
+ "gettingResultTime" : 0
}, {
"taskId" : 11,
"index" : 11,
@@ -89,7 +92,10 @@
"writeTime" : 94507,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 8,
+ "gettingResultTime" : 0
}, {
"taskId" : 12,
"index" : 12,
@@ -135,7 +141,10 @@
"writeTime" : 102476,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 7,
+ "gettingResultTime" : 0
}, {
"taskId" : 13,
"index" : 13,
@@ -181,7 +190,10 @@
"writeTime" : 95004,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 53,
+ "gettingResultTime" : 0
}, {
"taskId" : 14,
"index" : 14,
@@ -227,7 +239,10 @@
"writeTime" : 95646,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
}, {
"taskId" : 15,
"index" : 15,
@@ -273,7 +288,10 @@
"writeTime" : 602780,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 16,
"index" : 16,
@@ -319,7 +337,10 @@
"writeTime" : 108320,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 17,
"index" : 17,
@@ -365,7 +386,10 @@
"writeTime" : 99944,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 20,
+ "gettingResultTime" : 0
}, {
"taskId" : 18,
"index" : 18,
@@ -411,7 +435,10 @@
"writeTime" : 100836,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
}, {
"taskId" : 19,
"index" : 19,
@@ -457,7 +484,10 @@
"writeTime" : 95788,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
}, {
"taskId" : 20,
"index" : 20,
@@ -503,7 +533,10 @@
"writeTime" : 97716,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 21,
"index" : 21,
@@ -549,7 +582,10 @@
"writeTime" : 100270,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
}, {
"taskId" : 22,
"index" : 22,
@@ -595,7 +631,10 @@
"writeTime" : 143427,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
}, {
"taskId" : 23,
"index" : 23,
@@ -641,7 +680,10 @@
"writeTime" : 91844,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 16,
+ "gettingResultTime" : 0
}, {
"taskId" : 24,
"index" : 24,
@@ -687,7 +729,10 @@
"writeTime" : 157194,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
}, {
"taskId" : 25,
"index" : 25,
@@ -733,7 +778,10 @@
"writeTime" : 94134,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 9,
+ "gettingResultTime" : 0
}, {
"taskId" : 26,
"index" : 26,
@@ -779,7 +827,10 @@
"writeTime" : 108213,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 8,
+ "gettingResultTime" : 0
}, {
"taskId" : 27,
"index" : 27,
@@ -825,7 +876,10 @@
"writeTime" : 102019,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 6,
+ "gettingResultTime" : 0
}, {
"taskId" : 28,
"index" : 28,
@@ -871,7 +925,10 @@
"writeTime" : 104299,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 17,
+ "gettingResultTime" : 0
}, {
"taskId" : 29,
"index" : 29,
@@ -917,7 +974,10 @@
"writeTime" : 114938,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 11,
+ "gettingResultTime" : 0
}, {
"taskId" : 30,
"index" : 30,
@@ -963,7 +1023,10 @@
"writeTime" : 119770,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 24,
+ "gettingResultTime" : 0
}, {
"taskId" : 31,
"index" : 31,
@@ -1009,7 +1072,10 @@
"writeTime" : 92619,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 14,
+ "gettingResultTime" : 0
}, {
"taskId" : 32,
"index" : 32,
@@ -1055,7 +1121,10 @@
"writeTime" : 89603,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 13,
+ "gettingResultTime" : 0
}, {
"taskId" : 33,
"index" : 33,
@@ -1101,7 +1170,10 @@
"writeTime" : 118329,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 10,
+ "gettingResultTime" : 0
}, {
"taskId" : 34,
"index" : 34,
@@ -1147,7 +1219,10 @@
"writeTime" : 127746,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 9,
+ "gettingResultTime" : 0
}, {
"taskId" : 35,
"index" : 35,
@@ -1193,7 +1268,10 @@
"writeTime" : 160963,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 10,
+ "gettingResultTime" : 0
}, {
"taskId" : 36,
"index" : 36,
@@ -1239,7 +1317,10 @@
"writeTime" : 123855,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 6,
+ "gettingResultTime" : 0
}, {
"taskId" : 37,
"index" : 37,
@@ -1285,7 +1366,10 @@
"writeTime" : 111869,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 6,
+ "gettingResultTime" : 0
}, {
"taskId" : 38,
"index" : 38,
@@ -1331,7 +1415,10 @@
"writeTime" : 131158,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 13,
+ "gettingResultTime" : 0
}, {
"taskId" : 39,
"index" : 39,
@@ -1377,7 +1464,10 @@
"writeTime" : 98748,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 13,
+ "gettingResultTime" : 0
}, {
"taskId" : 40,
"index" : 40,
@@ -1423,7 +1513,10 @@
"writeTime" : 94792,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 6,
+ "gettingResultTime" : 0
}, {
"taskId" : 41,
"index" : 41,
@@ -1469,7 +1562,10 @@
"writeTime" : 90765,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 6,
+ "gettingResultTime" : 0
}, {
"taskId" : 42,
"index" : 42,
@@ -1515,7 +1611,10 @@
"writeTime" : 103713,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 15,
+ "gettingResultTime" : 0
}, {
"taskId" : 43,
"index" : 43,
@@ -1561,7 +1660,10 @@
"writeTime" : 171516,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 21,
+ "gettingResultTime" : 0
}, {
"taskId" : 44,
"index" : 44,
@@ -1607,7 +1709,10 @@
"writeTime" : 98293,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 16,
+ "gettingResultTime" : 0
}, {
"taskId" : 45,
"index" : 45,
@@ -1653,7 +1758,10 @@
"writeTime" : 92985,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 16,
+ "gettingResultTime" : 0
}, {
"taskId" : 46,
"index" : 46,
@@ -1699,7 +1807,10 @@
"writeTime" : 113322,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 11,
+ "gettingResultTime" : 0
}, {
"taskId" : 47,
"index" : 47,
@@ -1745,7 +1856,10 @@
"writeTime" : 103015,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 13,
+ "gettingResultTime" : 0
}, {
"taskId" : 48,
"index" : 48,
@@ -1791,7 +1905,10 @@
"writeTime" : 139844,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
}, {
"taskId" : 49,
"index" : 49,
@@ -1837,7 +1954,10 @@
"writeTime" : 94984,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 50,
"index" : 50,
@@ -1883,7 +2003,10 @@
"writeTime" : 90836,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 51,
"index" : 51,
@@ -1929,7 +2052,10 @@
"writeTime" : 96013,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 2,
+ "gettingResultTime" : 0
}, {
"taskId" : 52,
"index" : 52,
@@ -1975,7 +2101,10 @@
"writeTime" : 89664,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
}, {
"taskId" : 53,
"index" : 53,
@@ -2021,7 +2150,10 @@
"writeTime" : 92835,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
}, {
"taskId" : 54,
"index" : 54,
@@ -2067,7 +2199,10 @@
"writeTime" : 90506,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 38,
+ "gettingResultTime" : 0
}, {
"taskId" : 55,
"index" : 55,
@@ -2113,7 +2248,10 @@
"writeTime" : 108309,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
}, {
"taskId" : 56,
"index" : 56,
@@ -2159,7 +2297,10 @@
"writeTime" : 90329,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 6,
+ "gettingResultTime" : 0
}, {
"taskId" : 57,
"index" : 57,
@@ -2205,7 +2346,10 @@
"writeTime" : 96849,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 2,
+ "gettingResultTime" : 0
}, {
"taskId" : 58,
"index" : 58,
@@ -2251,7 +2395,10 @@
"writeTime" : 97521,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 59,
"index" : 59,
@@ -2297,5 +2444,8 @@
"writeTime" : 100753,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
index bc1cd49909d31..82e339c8f56dd 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
@@ -43,7 +43,10 @@
"writeTime" : 4016617,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 38,
+ "gettingResultTime" : 0
}, {
"taskId" : 5,
"index" : 5,
@@ -89,7 +92,10 @@
"writeTime" : 3675510,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 33,
+ "gettingResultTime" : 0
}, {
"taskId" : 1,
"index" : 1,
@@ -135,7 +141,10 @@
"writeTime" : 3934399,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 40,
+ "gettingResultTime" : 0
}, {
"taskId" : 7,
"index" : 7,
@@ -181,7 +190,10 @@
"writeTime" : 2579051,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 43,
+ "gettingResultTime" : 0
}, {
"taskId" : 4,
"index" : 4,
@@ -227,7 +239,10 @@
"writeTime" : 83022,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 38,
+ "gettingResultTime" : 0
}, {
"taskId" : 3,
"index" : 3,
@@ -273,7 +288,10 @@
"writeTime" : 1311694,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 41,
+ "gettingResultTime" : 0
}, {
"taskId" : 0,
"index" : 0,
@@ -319,7 +337,10 @@
"writeTime" : 3842811,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 53,
+ "gettingResultTime" : 0
}, {
"taskId" : 2,
"index" : 2,
@@ -365,7 +386,10 @@
"writeTime" : 89885,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 37,
+ "gettingResultTime" : 0
}, {
"taskId" : 22,
"index" : 22,
@@ -411,7 +435,10 @@
"writeTime" : 143427,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
}, {
"taskId" : 18,
"index" : 18,
@@ -457,7 +484,10 @@
"writeTime" : 100836,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
}, {
"taskId" : 17,
"index" : 17,
@@ -503,7 +533,10 @@
"writeTime" : 99944,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 20,
+ "gettingResultTime" : 0
}, {
"taskId" : 21,
"index" : 21,
@@ -549,7 +582,10 @@
"writeTime" : 100270,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
}, {
"taskId" : 19,
"index" : 19,
@@ -595,7 +631,10 @@
"writeTime" : 95788,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
}, {
"taskId" : 16,
"index" : 16,
@@ -641,7 +680,10 @@
"writeTime" : 108320,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 9,
"index" : 9,
@@ -687,7 +729,10 @@
"writeTime" : 101664,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 8,
+ "gettingResultTime" : 0
}, {
"taskId" : 20,
"index" : 20,
@@ -733,7 +778,10 @@
"writeTime" : 97716,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 14,
"index" : 14,
@@ -779,7 +827,10 @@
"writeTime" : 95646,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
}, {
"taskId" : 8,
"index" : 8,
@@ -825,7 +876,10 @@
"writeTime" : 121551,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
}, {
"taskId" : 12,
"index" : 12,
@@ -871,7 +925,10 @@
"writeTime" : 102476,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 7,
+ "gettingResultTime" : 0
}, {
"taskId" : 15,
"index" : 15,
@@ -917,5 +974,8 @@
"writeTime" : 602780,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
index bc1cd49909d31..82e339c8f56dd 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
@@ -43,7 +43,10 @@
"writeTime" : 4016617,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 38,
+ "gettingResultTime" : 0
}, {
"taskId" : 5,
"index" : 5,
@@ -89,7 +92,10 @@
"writeTime" : 3675510,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 33,
+ "gettingResultTime" : 0
}, {
"taskId" : 1,
"index" : 1,
@@ -135,7 +141,10 @@
"writeTime" : 3934399,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 40,
+ "gettingResultTime" : 0
}, {
"taskId" : 7,
"index" : 7,
@@ -181,7 +190,10 @@
"writeTime" : 2579051,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 43,
+ "gettingResultTime" : 0
}, {
"taskId" : 4,
"index" : 4,
@@ -227,7 +239,10 @@
"writeTime" : 83022,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 38,
+ "gettingResultTime" : 0
}, {
"taskId" : 3,
"index" : 3,
@@ -273,7 +288,10 @@
"writeTime" : 1311694,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 41,
+ "gettingResultTime" : 0
}, {
"taskId" : 0,
"index" : 0,
@@ -319,7 +337,10 @@
"writeTime" : 3842811,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 53,
+ "gettingResultTime" : 0
}, {
"taskId" : 2,
"index" : 2,
@@ -365,7 +386,10 @@
"writeTime" : 89885,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 37,
+ "gettingResultTime" : 0
}, {
"taskId" : 22,
"index" : 22,
@@ -411,7 +435,10 @@
"writeTime" : 143427,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
}, {
"taskId" : 18,
"index" : 18,
@@ -457,7 +484,10 @@
"writeTime" : 100836,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
}, {
"taskId" : 17,
"index" : 17,
@@ -503,7 +533,10 @@
"writeTime" : 99944,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 20,
+ "gettingResultTime" : 0
}, {
"taskId" : 21,
"index" : 21,
@@ -549,7 +582,10 @@
"writeTime" : 100270,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
}, {
"taskId" : 19,
"index" : 19,
@@ -595,7 +631,10 @@
"writeTime" : 95788,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
}, {
"taskId" : 16,
"index" : 16,
@@ -641,7 +680,10 @@
"writeTime" : 108320,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 9,
"index" : 9,
@@ -687,7 +729,10 @@
"writeTime" : 101664,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 8,
+ "gettingResultTime" : 0
}, {
"taskId" : 20,
"index" : 20,
@@ -733,7 +778,10 @@
"writeTime" : 97716,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 14,
"index" : 14,
@@ -779,7 +827,10 @@
"writeTime" : 95646,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
}, {
"taskId" : 8,
"index" : 8,
@@ -825,7 +876,10 @@
"writeTime" : 121551,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
}, {
"taskId" : 12,
"index" : 12,
@@ -871,7 +925,10 @@
"writeTime" : 102476,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 7,
+ "gettingResultTime" : 0
}, {
"taskId" : 15,
"index" : 15,
@@ -917,5 +974,8 @@
"writeTime" : 602780,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
index 09857cb401acd..01eef1b565bf6 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
@@ -43,7 +43,10 @@
"writeTime" : 94792,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 6,
+ "gettingResultTime" : 0
}, {
"taskId" : 41,
"index" : 41,
@@ -89,7 +92,10 @@
"writeTime" : 90765,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 6,
+ "gettingResultTime" : 0
}, {
"taskId" : 43,
"index" : 43,
@@ -135,7 +141,10 @@
"writeTime" : 171516,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 21,
+ "gettingResultTime" : 0
}, {
"taskId" : 57,
"index" : 57,
@@ -181,7 +190,10 @@
"writeTime" : 96849,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 2,
+ "gettingResultTime" : 0
}, {
"taskId" : 58,
"index" : 58,
@@ -227,7 +239,10 @@
"writeTime" : 97521,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 68,
"index" : 68,
@@ -273,7 +288,10 @@
"writeTime" : 101750,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 86,
"index" : 86,
@@ -319,7 +337,10 @@
"writeTime" : 95848,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 8,
+ "gettingResultTime" : 0
}, {
"taskId" : 32,
"index" : 32,
@@ -365,7 +386,10 @@
"writeTime" : 89603,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 13,
+ "gettingResultTime" : 0
}, {
"taskId" : 39,
"index" : 39,
@@ -411,7 +435,10 @@
"writeTime" : 98748,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 13,
+ "gettingResultTime" : 0
}, {
"taskId" : 42,
"index" : 42,
@@ -457,7 +484,10 @@
"writeTime" : 103713,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 15,
+ "gettingResultTime" : 0
}, {
"taskId" : 51,
"index" : 51,
@@ -503,7 +533,10 @@
"writeTime" : 96013,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 2,
+ "gettingResultTime" : 0
}, {
"taskId" : 59,
"index" : 59,
@@ -549,7 +582,10 @@
"writeTime" : 100753,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
}, {
"taskId" : 63,
"index" : 63,
@@ -595,7 +631,10 @@
"writeTime" : 102779,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
}, {
"taskId" : 87,
"index" : 87,
@@ -641,7 +680,10 @@
"writeTime" : 102159,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 7,
+ "gettingResultTime" : 0
}, {
"taskId" : 90,
"index" : 90,
@@ -687,7 +729,10 @@
"writeTime" : 98472,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 99,
"index" : 99,
@@ -733,7 +778,10 @@
"writeTime" : 133964,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
}, {
"taskId" : 44,
"index" : 44,
@@ -779,7 +827,10 @@
"writeTime" : 98293,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 16,
+ "gettingResultTime" : 0
}, {
"taskId" : 47,
"index" : 47,
@@ -825,7 +876,10 @@
"writeTime" : 103015,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 13,
+ "gettingResultTime" : 0
}, {
"taskId" : 50,
"index" : 50,
@@ -871,7 +925,10 @@
"writeTime" : 90836,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
}, {
"taskId" : 52,
"index" : 52,
@@ -917,5 +974,8 @@
"writeTime" : 89664,
"recordsWritten" : 10
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
index 963f010968b62..a8e1fd303a42a 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
@@ -83,14 +83,17 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 30,
+ "gettingResultTime" : 0
},
- "1" : {
- "taskId" : 1,
- "index" : 1,
+ "5" : {
+ "taskId" : 5,
+ "index" : 5,
"attempt" : 0,
- "launchTime" : "2015-03-16T19:25:36.521GMT",
- "duration" : 53,
+ "launchTime" : "2015-03-16T19:25:36.523GMT",
+ "duration" : 52,
"executorId" : "",
"host" : "localhost",
"status" : "SUCCESS",
@@ -99,11 +102,11 @@
"accumulatorUpdates" : [ {
"id" : 1,
"name" : "my counter",
- "update" : "247",
- "value" : "2175"
+ "update" : "897",
+ "value" : "3750"
} ],
"taskMetrics" : {
- "executorDeserializeTime" : 14,
+ "executorDeserializeTime" : 12,
"executorDeserializeCpuTime" : 0,
"executorRunTime" : 15,
"executorCpuTime" : 0,
@@ -135,14 +138,17 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 23,
+ "gettingResultTime" : 0
},
- "2" : {
- "taskId" : 2,
- "index" : 2,
+ "1" : {
+ "taskId" : 1,
+ "index" : 1,
"attempt" : 0,
- "launchTime" : "2015-03-16T19:25:36.522GMT",
- "duration" : 48,
+ "launchTime" : "2015-03-16T19:25:36.521GMT",
+ "duration" : 53,
"executorId" : "",
"host" : "localhost",
"status" : "SUCCESS",
@@ -151,11 +157,11 @@
"accumulatorUpdates" : [ {
"id" : 1,
"name" : "my counter",
- "update" : "378",
- "value" : "378"
+ "update" : "247",
+ "value" : "2175"
} ],
"taskMetrics" : {
- "executorDeserializeTime" : 13,
+ "executorDeserializeTime" : 14,
"executorDeserializeCpuTime" : 0,
"executorRunTime" : 15,
"executorCpuTime" : 0,
@@ -187,14 +193,17 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 22,
+ "gettingResultTime" : 0
},
- "3" : {
- "taskId" : 3,
- "index" : 3,
+ "6" : {
+ "taskId" : 6,
+ "index" : 6,
"attempt" : 0,
- "launchTime" : "2015-03-16T19:25:36.522GMT",
- "duration" : 50,
+ "launchTime" : "2015-03-16T19:25:36.523GMT",
+ "duration" : 51,
"executorId" : "",
"host" : "localhost",
"status" : "SUCCESS",
@@ -203,11 +212,11 @@
"accumulatorUpdates" : [ {
"id" : 1,
"name" : "my counter",
- "update" : "572",
- "value" : "950"
+ "update" : "978",
+ "value" : "1928"
} ],
"taskMetrics" : {
- "executorDeserializeTime" : 13,
+ "executorDeserializeTime" : 12,
"executorDeserializeCpuTime" : 0,
"executorRunTime" : 15,
"executorCpuTime" : 0,
@@ -239,14 +248,17 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 22,
+ "gettingResultTime" : 0
},
- "4" : {
- "taskId" : 4,
- "index" : 4,
+ "2" : {
+ "taskId" : 2,
+ "index" : 2,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.522GMT",
- "duration" : 52,
+ "duration" : 48,
"executorId" : "",
"host" : "localhost",
"status" : "SUCCESS",
@@ -255,17 +267,17 @@
"accumulatorUpdates" : [ {
"id" : 1,
"name" : "my counter",
- "update" : "678",
- "value" : "2853"
+ "update" : "378",
+ "value" : "378"
} ],
"taskMetrics" : {
- "executorDeserializeTime" : 12,
+ "executorDeserializeTime" : 13,
"executorDeserializeCpuTime" : 0,
"executorRunTime" : 15,
"executorCpuTime" : 0,
"resultSize" : 697,
"jvmGcTime" : 0,
- "resultSerializationTime" : 1,
+ "resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
@@ -291,14 +303,17 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 18,
+ "gettingResultTime" : 0
},
- "5" : {
- "taskId" : 5,
- "index" : 5,
+ "7" : {
+ "taskId" : 7,
+ "index" : 7,
"attempt" : 0,
- "launchTime" : "2015-03-16T19:25:36.523GMT",
- "duration" : 52,
+ "launchTime" : "2015-03-16T19:25:36.524GMT",
+ "duration" : 51,
"executorId" : "",
"host" : "localhost",
"status" : "SUCCESS",
@@ -307,8 +322,8 @@
"accumulatorUpdates" : [ {
"id" : 1,
"name" : "my counter",
- "update" : "897",
- "value" : "3750"
+ "update" : "1222",
+ "value" : "4972"
} ],
"taskMetrics" : {
"executorDeserializeTime" : 12,
@@ -343,14 +358,17 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 22,
+ "gettingResultTime" : 0
},
- "6" : {
- "taskId" : 6,
- "index" : 6,
+ "3" : {
+ "taskId" : 3,
+ "index" : 3,
"attempt" : 0,
- "launchTime" : "2015-03-16T19:25:36.523GMT",
- "duration" : 51,
+ "launchTime" : "2015-03-16T19:25:36.522GMT",
+ "duration" : 50,
"executorId" : "",
"host" : "localhost",
"status" : "SUCCESS",
@@ -359,11 +377,11 @@
"accumulatorUpdates" : [ {
"id" : 1,
"name" : "my counter",
- "update" : "978",
- "value" : "1928"
+ "update" : "572",
+ "value" : "950"
} ],
"taskMetrics" : {
- "executorDeserializeTime" : 12,
+ "executorDeserializeTime" : 13,
"executorDeserializeCpuTime" : 0,
"executorRunTime" : 15,
"executorCpuTime" : 0,
@@ -395,14 +413,17 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 20,
+ "gettingResultTime" : 0
},
- "7" : {
- "taskId" : 7,
- "index" : 7,
+ "4" : {
+ "taskId" : 4,
+ "index" : 4,
"attempt" : 0,
- "launchTime" : "2015-03-16T19:25:36.524GMT",
- "duration" : 51,
+ "launchTime" : "2015-03-16T19:25:36.522GMT",
+ "duration" : 52,
"executorId" : "",
"host" : "localhost",
"status" : "SUCCESS",
@@ -411,8 +432,8 @@
"accumulatorUpdates" : [ {
"id" : 1,
"name" : "my counter",
- "update" : "1222",
- "value" : "4972"
+ "update" : "678",
+ "value" : "2853"
} ],
"taskMetrics" : {
"executorDeserializeTime" : 12,
@@ -421,7 +442,7 @@
"executorCpuTime" : 0,
"resultSize" : 697,
"jvmGcTime" : 0,
- "resultSerializationTime" : 2,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
@@ -447,7 +468,10 @@
"writeTime" : 0,
"recordsWritten" : 0
}
- }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 24,
+ "gettingResultTime" : 0
}
},
"executorSummary" : {
diff --git a/core/src/test/scala/org/apache/spark/FailureSuite.scala b/core/src/test/scala/org/apache/spark/FailureSuite.scala
index d805c67714ff8..f2d97d452ddb0 100644
--- a/core/src/test/scala/org/apache/spark/FailureSuite.scala
+++ b/core/src/test/scala/org/apache/spark/FailureSuite.scala
@@ -257,7 +257,9 @@ class FailureSuite extends SparkFunSuite with LocalSparkContext {
sc = new SparkContext("local[1,2]", "test")
intercept[SparkException] {
sc.parallelize(1 to 2).foreach { i =>
+ // scalastyle:off throwerror
throw new LinkageError()
+ // scalastyle:on throwerror
}
}
}
diff --git a/core/src/test/scala/org/apache/spark/FileSuite.scala b/core/src/test/scala/org/apache/spark/FileSuite.scala
index 34efcdf4bc886..df04a5ea1d99e 100644
--- a/core/src/test/scala/org/apache/spark/FileSuite.scala
+++ b/core/src/test/scala/org/apache/spark/FileSuite.scala
@@ -202,7 +202,7 @@ class FileSuite extends SparkFunSuite with LocalSparkContext {
sc = new SparkContext("local", "test")
val objs = sc.makeRDD(1 to 3).map { x =>
val loader = Thread.currentThread().getContextClassLoader
- Class.forName(className, true, loader).newInstance()
+ Class.forName(className, true, loader).getConstructor().newInstance()
}
val outputDir = new File(tempDir, "output").getAbsolutePath
objs.saveAsObjectFile(outputDir)
diff --git a/core/src/test/scala/org/apache/spark/ShuffleSuite.scala b/core/src/test/scala/org/apache/spark/ShuffleSuite.scala
index b917469e48747..35f728cd57fe2 100644
--- a/core/src/test/scala/org/apache/spark/ShuffleSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ShuffleSuite.scala
@@ -362,15 +362,19 @@ abstract class ShuffleSuite extends SparkFunSuite with Matchers with LocalSparkC
mapTrackerMaster.registerShuffle(0, 1)
// first attempt -- its successful
- val writer1 = manager.getWriter[Int, Int](shuffleHandle, 0,
- new TaskContextImpl(0, 0, 0, 0L, 0, taskMemoryManager, new Properties, metricsSystem))
+ val context1 =
+ new TaskContextImpl(0, 0, 0, 0L, 0, taskMemoryManager, new Properties, metricsSystem)
+ val writer1 = manager.getWriter[Int, Int](
+ shuffleHandle, 0, context1, context1.taskMetrics.shuffleWriteMetrics)
val data1 = (1 to 10).map { x => x -> x}
// second attempt -- also successful. We'll write out different data,
// just to simulate the fact that the records may get written differently
// depending on what gets spilled, what gets combined, etc.
- val writer2 = manager.getWriter[Int, Int](shuffleHandle, 0,
- new TaskContextImpl(0, 0, 0, 1L, 0, taskMemoryManager, new Properties, metricsSystem))
+ val context2 =
+ new TaskContextImpl(0, 0, 0, 1L, 0, taskMemoryManager, new Properties, metricsSystem)
+ val writer2 = manager.getWriter[Int, Int](
+ shuffleHandle, 0, context2, context2.taskMetrics.shuffleWriteMetrics)
val data2 = (11 to 20).map { x => x -> x}
// interleave writes of both attempts -- we want to test that both attempts can occur
@@ -397,8 +401,10 @@ abstract class ShuffleSuite extends SparkFunSuite with Matchers with LocalSparkC
mapTrackerMaster.registerMapOutput(0, 0, mapStatus)
}
- val reader = manager.getReader[Int, Int](shuffleHandle, 0, 1,
- new TaskContextImpl(1, 0, 0, 2L, 0, taskMemoryManager, new Properties, metricsSystem))
+ val taskContext = new TaskContextImpl(
+ 1, 0, 0, 2L, 0, taskMemoryManager, new Properties, metricsSystem)
+ val metrics = taskContext.taskMetrics.createTempShuffleReadMetrics()
+ val reader = manager.getReader[Int, Int](shuffleHandle, 0, 1, taskContext, metrics)
val readData = reader.read().toIndexedSeq
assert(readData === data1.toIndexedSeq || readData === data2.toIndexedSeq)
diff --git a/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala b/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala
index 24e596e1ecdaf..a6666db4e95c3 100644
--- a/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala
+++ b/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala
@@ -58,5 +58,12 @@ abstract class BenchmarkBase {
o.close()
}
}
+
+ afterAll()
}
+
+ /**
+ * Any shutdown code to ensure a clean shutdown
+ */
+ def afterAll(): Unit = {}
}
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index 652c36ffa6e71..c093789244bfe 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -962,6 +962,25 @@ class SparkSubmitSuite
}
}
+ test("remove copies of application jar from classpath") {
+ val fs = File.separator
+ val sparkConf = new SparkConf(false)
+ val hadoopConf = new Configuration()
+ val secMgr = new SecurityManager(sparkConf)
+
+ val appJarName = "myApp.jar"
+ val jar1Name = "myJar1.jar"
+ val jar2Name = "myJar2.jar"
+ val userJar = s"file:/path${fs}to${fs}app${fs}jar$fs$appJarName"
+ val jars = s"file:/$jar1Name,file:/$appJarName,file:/$jar2Name"
+
+ val resolvedJars = DependencyUtils
+ .resolveAndDownloadJars(jars, userJar, sparkConf, hadoopConf, secMgr)
+
+ assert(!resolvedJars.contains(appJarName))
+ assert(resolvedJars.contains(jar1Name) && resolvedJars.contains(jar2Name))
+ }
+
test("Avoid re-upload remote resources in yarn client mode") {
val hadoopConf = new Configuration()
updateConfWithFakeS3Fs(hadoopConf)
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
index 87d585278a747..b0ced46c9c7b8 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
@@ -23,10 +23,12 @@ import java.util.{Date, Locale}
import java.util.concurrent.TimeUnit
import java.util.zip.{ZipInputStream, ZipOutputStream}
+import scala.collection.JavaConverters._
import scala.concurrent.duration._
import scala.language.postfixOps
import com.google.common.io.{ByteStreams, Files}
+import org.apache.commons.io.FileUtils
import org.apache.hadoop.fs.{FileStatus, FileSystem, FSDataInputStream, Path}
import org.apache.hadoop.hdfs.{DFSInputStream, DistributedFileSystem}
import org.apache.hadoop.security.AccessControlException
@@ -40,6 +42,7 @@ import org.scalatest.concurrent.Eventually._
import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.DRIVER_LOG_DFS_DIR
import org.apache.spark.internal.config.History._
import org.apache.spark.io._
import org.apache.spark.scheduler._
@@ -47,6 +50,7 @@ import org.apache.spark.security.GroupMappingServiceProvider
import org.apache.spark.status.AppStatusStore
import org.apache.spark.status.api.v1.{ApplicationAttemptInfo, ApplicationInfo}
import org.apache.spark.util.{Clock, JsonProtocol, ManualClock, Utils}
+import org.apache.spark.util.logging.DriverLogger
class FsHistoryProviderSuite extends SparkFunSuite with BeforeAndAfter with Matchers with Logging {
@@ -413,6 +417,63 @@ class FsHistoryProviderSuite extends SparkFunSuite with BeforeAndAfter with Matc
}
}
+ test("driver log cleaner") {
+ val firstFileModifiedTime = TimeUnit.SECONDS.toMillis(10)
+ val secondFileModifiedTime = TimeUnit.SECONDS.toMillis(20)
+ val maxAge = TimeUnit.SECONDS.toSeconds(40)
+ val clock = new ManualClock(0)
+ val testConf = new SparkConf()
+ testConf.set("spark.history.fs.logDirectory",
+ Utils.createTempDir(namePrefix = "eventLog").getAbsolutePath())
+ testConf.set(DRIVER_LOG_DFS_DIR, testDir.getAbsolutePath())
+ testConf.set(DRIVER_LOG_CLEANER_ENABLED, true)
+ testConf.set(DRIVER_LOG_CLEANER_INTERVAL, maxAge / 4)
+ testConf.set(MAX_DRIVER_LOG_AGE_S, maxAge)
+ val provider = new FsHistoryProvider(testConf, clock)
+
+ val log1 = FileUtils.getFile(testDir, "1" + DriverLogger.DRIVER_LOG_FILE_SUFFIX)
+ createEmptyFile(log1)
+ clock.setTime(firstFileModifiedTime)
+ log1.setLastModified(clock.getTimeMillis())
+ provider.cleanDriverLogs()
+
+ val log2 = FileUtils.getFile(testDir, "2" + DriverLogger.DRIVER_LOG_FILE_SUFFIX)
+ createEmptyFile(log2)
+ val log3 = FileUtils.getFile(testDir, "3" + DriverLogger.DRIVER_LOG_FILE_SUFFIX)
+ createEmptyFile(log3)
+ clock.setTime(secondFileModifiedTime)
+ log2.setLastModified(clock.getTimeMillis())
+ log3.setLastModified(clock.getTimeMillis())
+ // This should not trigger any cleanup
+ provider.cleanDriverLogs()
+ provider.listing.view(classOf[LogInfo]).iterator().asScala.toSeq.size should be(3)
+
+ // Should trigger cleanup for first file but not second one
+ clock.setTime(firstFileModifiedTime + TimeUnit.SECONDS.toMillis(maxAge) + 1)
+ provider.cleanDriverLogs()
+ provider.listing.view(classOf[LogInfo]).iterator().asScala.toSeq.size should be(2)
+ assert(!log1.exists())
+ assert(log2.exists())
+ assert(log3.exists())
+
+ // Update the third file length while keeping the original modified time
+ Files.write("Add logs to file".getBytes(), log3)
+ log3.setLastModified(secondFileModifiedTime)
+ // Should cleanup the second file but not the third file, as filelength changed.
+ clock.setTime(secondFileModifiedTime + TimeUnit.SECONDS.toMillis(maxAge) + 1)
+ provider.cleanDriverLogs()
+ provider.listing.view(classOf[LogInfo]).iterator().asScala.toSeq.size should be(1)
+ assert(!log1.exists())
+ assert(!log2.exists())
+ assert(log3.exists())
+
+ // Should cleanup the third file as well.
+ clock.setTime(secondFileModifiedTime + 2 * TimeUnit.SECONDS.toMillis(maxAge) + 2)
+ provider.cleanDriverLogs()
+ provider.listing.view(classOf[LogInfo]).iterator().asScala.toSeq.size should be(0)
+ assert(!log3.exists())
+ }
+
test("SPARK-8372: new logs with no app ID are ignored") {
val provider = new FsHistoryProvider(createTestConf())
@@ -869,16 +930,19 @@ class FsHistoryProviderSuite extends SparkFunSuite with BeforeAndAfter with Matc
when(dfsIn.getFileLength).thenReturn(200)
// FileStatus.getLen is more than logInfo fileSize
var fileStatus = new FileStatus(200, false, 0, 0, 0, path)
- var logInfo = new LogInfo(path.toString, 0, Some("appId"), Some("attemptId"), 100)
+ var logInfo = new LogInfo(path.toString, 0, LogType.EventLogs, Some("appId"),
+ Some("attemptId"), 100)
assert(mockedProvider.shouldReloadLog(logInfo, fileStatus))
fileStatus = new FileStatus()
fileStatus.setPath(path)
// DFSInputStream.getFileLength is more than logInfo fileSize
- logInfo = new LogInfo(path.toString, 0, Some("appId"), Some("attemptId"), 100)
+ logInfo = new LogInfo(path.toString, 0, LogType.EventLogs, Some("appId"),
+ Some("attemptId"), 100)
assert(mockedProvider.shouldReloadLog(logInfo, fileStatus))
// DFSInputStream.getFileLength is equal to logInfo fileSize
- logInfo = new LogInfo(path.toString, 0, Some("appId"), Some("attemptId"), 200)
+ logInfo = new LogInfo(path.toString, 0, LogType.EventLogs, Some("appId"),
+ Some("attemptId"), 200)
assert(!mockedProvider.shouldReloadLog(logInfo, fileStatus))
// in.getWrappedStream returns other than DFSInputStream
val bin = mock(classOf[BufferedInputStream])
diff --git a/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala
index 4839c842cc785..89b8bb4ff7d03 100644
--- a/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala
@@ -396,6 +396,18 @@ class StandaloneRestSubmitSuite extends SparkFunSuite with BeforeAndAfterEach {
assert(filteredVariables == Map("SPARK_VAR" -> "1"))
}
+ test("client does not send 'SPARK_HOME' env var by default") {
+ val environmentVariables = Map("SPARK_VAR" -> "1", "SPARK_HOME" -> "1")
+ val filteredVariables = RestSubmissionClient.filterSystemEnvironment(environmentVariables)
+ assert(filteredVariables == Map("SPARK_VAR" -> "1"))
+ }
+
+ test("client does not send 'SPARK_CONF_DIR' env var by default") {
+ val environmentVariables = Map("SPARK_VAR" -> "1", "SPARK_CONF_DIR" -> "1")
+ val filteredVariables = RestSubmissionClient.filterSystemEnvironment(environmentVariables)
+ assert(filteredVariables == Map("SPARK_VAR" -> "1"))
+ }
+
test("client includes mesos env vars") {
val environmentVariables = Map("SPARK_VAR" -> "1", "MESOS_VAR" -> "1", "OTHER_VAR" -> "1")
val filteredVariables = RestSubmissionClient.filterSystemEnvironment(environmentVariables)
diff --git a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
index 1f8a65707b2f7..32a94e60484e3 100644
--- a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
@@ -467,7 +467,9 @@ class FetchFailureHidingRDD(
} catch {
case t: Throwable =>
if (throwOOM) {
+ // scalastyle:off throwerror
throw new OutOfMemoryError("OOM while handling another exception")
+ // scalastyle:on throwerror
} else if (interrupt) {
// make sure our test is setup correctly
assert(TaskContext.get().asInstanceOf[TaskContextImpl].fetchFailed.isDefined)
diff --git a/core/src/test/scala/org/apache/spark/io/ChunkedByteBufferSuite.scala b/core/src/test/scala/org/apache/spark/io/ChunkedByteBufferSuite.scala
index ff117b1c21cb1..083c5e696b753 100644
--- a/core/src/test/scala/org/apache/spark/io/ChunkedByteBufferSuite.scala
+++ b/core/src/test/scala/org/apache/spark/io/ChunkedByteBufferSuite.scala
@@ -90,7 +90,7 @@ class ChunkedByteBufferSuite extends SparkFunSuite with SharedSparkContext {
val empty = ByteBuffer.wrap(Array.empty[Byte])
val bytes1 = ByteBuffer.wrap(Array.tabulate(256)(_.toByte))
val bytes2 = ByteBuffer.wrap(Array.tabulate(128)(_.toByte))
- val chunkedByteBuffer = new ChunkedByteBuffer(Array(empty, bytes1, bytes2))
+ val chunkedByteBuffer = new ChunkedByteBuffer(Array(empty, bytes1, empty, bytes2))
assert(chunkedByteBuffer.size === bytes1.limit() + bytes2.limit())
val inputStream = chunkedByteBuffer.toInputStream(dispose = false)
diff --git a/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala b/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala
index 47af5c3320dd9..0ec359d1c94f3 100644
--- a/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala
@@ -574,7 +574,7 @@ class PairRDDFunctionsSuite extends SparkFunSuite with SharedSparkContext {
}
test("saveNewAPIHadoopFile should call setConf if format is configurable") {
- val pairs = sc.parallelize(Array((new Integer(1), new Integer(1))))
+ val pairs = sc.parallelize(Array((Integer.valueOf(1), Integer.valueOf(1))))
// No error, non-configurable formats still work
pairs.saveAsNewAPIHadoopFile[NewFakeFormat]("ignored")
@@ -591,14 +591,14 @@ class PairRDDFunctionsSuite extends SparkFunSuite with SharedSparkContext {
test("The JobId on the driver and executors should be the same during the commit") {
// Create more than one rdd to mimic stageId not equal to rddId
val pairs = sc.parallelize(Array((1, 2), (2, 3)), 2)
- .map { p => (new Integer(p._1 + 1), new Integer(p._2 + 1)) }
+ .map { p => (Integer.valueOf(p._1 + 1), Integer.valueOf(p._2 + 1)) }
.filter { p => p._1 > 0 }
pairs.saveAsNewAPIHadoopFile[YetAnotherFakeFormat]("ignored")
assert(JobID.jobid != -1)
}
test("saveAsHadoopFile should respect configured output committers") {
- val pairs = sc.parallelize(Array((new Integer(1), new Integer(1))))
+ val pairs = sc.parallelize(Array((Integer.valueOf(1), Integer.valueOf(1))))
val conf = new JobConf()
conf.setOutputCommitter(classOf[FakeOutputCommitter])
@@ -610,7 +610,7 @@ class PairRDDFunctionsSuite extends SparkFunSuite with SharedSparkContext {
}
test("failure callbacks should be called before calling writer.close() in saveNewAPIHadoopFile") {
- val pairs = sc.parallelize(Array((new Integer(1), new Integer(2))), 1)
+ val pairs = sc.parallelize(Array((Integer.valueOf(1), Integer.valueOf(2))), 1)
FakeWriterWithCallback.calledBy = ""
FakeWriterWithCallback.exception = null
@@ -625,7 +625,7 @@ class PairRDDFunctionsSuite extends SparkFunSuite with SharedSparkContext {
}
test("failure callbacks should be called before calling writer.close() in saveAsHadoopFile") {
- val pairs = sc.parallelize(Array((new Integer(1), new Integer(2))), 1)
+ val pairs = sc.parallelize(Array((Integer.valueOf(1), Integer.valueOf(2))), 1)
val conf = new JobConf()
FakeWriterWithCallback.calledBy = ""
@@ -643,7 +643,7 @@ class PairRDDFunctionsSuite extends SparkFunSuite with SharedSparkContext {
test("saveAsNewAPIHadoopDataset should support invalid output paths when " +
"there are no files to be committed to an absolute output location") {
- val pairs = sc.parallelize(Array((new Integer(1), new Integer(2))), 1)
+ val pairs = sc.parallelize(Array((Integer.valueOf(1), Integer.valueOf(2))), 1)
def saveRddWithPath(path: String): Unit = {
val job = NewJob.getInstance(new Configuration(sc.hadoopConfiguration))
@@ -671,7 +671,7 @@ class PairRDDFunctionsSuite extends SparkFunSuite with SharedSparkContext {
// for non-null invalid paths.
test("saveAsHadoopDataset should respect empty output directory when " +
"there are no files to be committed to an absolute output location") {
- val pairs = sc.parallelize(Array((new Integer(1), new Integer(2))), 1)
+ val pairs = sc.parallelize(Array((Integer.valueOf(1), Integer.valueOf(2))), 1)
val conf = new JobConf()
conf.setOutputKeyClass(classOf[Integer])
diff --git a/core/src/test/scala/org/apache/spark/rdd/ParallelCollectionSplitSuite.scala b/core/src/test/scala/org/apache/spark/rdd/ParallelCollectionSplitSuite.scala
index 31ce9483cf20a..424d9f825c465 100644
--- a/core/src/test/scala/org/apache/spark/rdd/ParallelCollectionSplitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/ParallelCollectionSplitSuite.scala
@@ -215,7 +215,7 @@ class ParallelCollectionSplitSuite extends SparkFunSuite with Checkers {
}
test("exclusive ranges of doubles") {
- val data = 1.0 until 100.0 by 1.0
+ val data = Range.BigDecimal(1, 100, 1)
val slices = ParallelCollectionRDD.slice(data, 3)
assert(slices.size === 3)
assert(slices.map(_.size).sum === 99)
@@ -223,7 +223,7 @@ class ParallelCollectionSplitSuite extends SparkFunSuite with Checkers {
}
test("inclusive ranges of doubles") {
- val data = 1.0 to 100.0 by 1.0
+ val data = Range.BigDecimal.inclusive(1, 100, 1)
val slices = ParallelCollectionRDD.slice(data, 3)
assert(slices.size === 3)
assert(slices.map(_.size).sum === 100)
diff --git a/core/src/test/scala/org/apache/spark/scheduler/CustomShuffledRDD.scala b/core/src/test/scala/org/apache/spark/scheduler/CustomShuffledRDD.scala
index 838686923767e..1be2e2a067115 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/CustomShuffledRDD.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/CustomShuffledRDD.scala
@@ -104,8 +104,9 @@ class CustomShuffledRDD[K, V, C](
override def compute(p: Partition, context: TaskContext): Iterator[(K, C)] = {
val part = p.asInstanceOf[CustomShuffledRDDPartition]
+ val metrics = context.taskMetrics().createTempShuffleReadMetrics()
SparkEnv.get.shuffleManager.getReader(
- dependency.shuffleHandle, part.startIndexInParent, part.endIndexInParent, context)
+ dependency.shuffleHandle, part.startIndexInParent, part.endIndexInParent, context, metrics)
.read()
.asInstanceOf[Iterator[(K, C)]]
}
diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskResultGetterSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskResultGetterSuite.scala
index 1bddba8f6c82b..efb8b15cf6b4d 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/TaskResultGetterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/TaskResultGetterSuite.scala
@@ -194,7 +194,7 @@ class TaskResultGetterSuite extends SparkFunSuite with BeforeAndAfter with Local
// jar.
sc = new SparkContext("local", "test", conf)
val rdd = sc.parallelize(Seq(1), 1).map { _ =>
- val exc = excClass.newInstance().asInstanceOf[Exception]
+ val exc = excClass.getConstructor().newInstance().asInstanceOf[Exception]
throw exc
}
@@ -265,7 +265,9 @@ class TaskResultGetterSuite extends SparkFunSuite with BeforeAndAfter with Local
private class UndeserializableException extends Exception {
private def readObject(in: ObjectInputStream): Unit = {
+ // scalastyle:off throwerror
throw new NoClassDefFoundError()
+ // scalastyle:on throwerror
}
}
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerBenchmark.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerBenchmark.scala
new file mode 100644
index 0000000000000..2a15c6f6a2d96
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerBenchmark.scala
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.serializer
+
+import scala.concurrent._
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.duration._
+
+import org.apache.spark.{SparkConf, SparkContext}
+import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
+import org.apache.spark.serializer.KryoTest._
+import org.apache.spark.util.ThreadUtils
+
+/**
+ * Benchmark for KryoPool vs old "pool of 1".
+ * To run this benchmark:
+ * {{{
+ * 1. without sbt:
+ * bin/spark-submit --class --jars
+ * 2. build/sbt "core/test:runMain "
+ * 3. generate result:
+ * SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "core/test:runMain "
+ * Results will be written to "benchmarks/KryoSerializerBenchmark-results.txt".
+ * }}}
+ */
+object KryoSerializerBenchmark extends BenchmarkBase {
+
+ var sc: SparkContext = null
+ val N = 500
+ override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+ val name = "Benchmark KryoPool vs old\"pool of 1\" implementation"
+ runBenchmark(name) {
+ val benchmark = new Benchmark(name, N, 10, output = output)
+ Seq(true, false).foreach(usePool => run(usePool, benchmark))
+ benchmark.run()
+ }
+ }
+
+ private def run(usePool: Boolean, benchmark: Benchmark): Unit = {
+ lazy val sc = createSparkContext(usePool)
+
+ benchmark.addCase(s"KryoPool:$usePool") { _ =>
+ val futures = for (_ <- 0 until N) yield {
+ Future {
+ sc.parallelize(0 until 10).map(i => i + 1).count()
+ }
+ }
+
+ val future = Future.sequence(futures)
+
+ ThreadUtils.awaitResult(future, 10.minutes)
+ }
+ }
+
+ def createSparkContext(usePool: Boolean): SparkContext = {
+ val conf = new SparkConf()
+ conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
+ conf.set("spark.kryo.registrator", classOf[MyRegistrator].getName)
+ conf.set("spark.kryo.pool", usePool.toString)
+
+ if (sc != null) {
+ sc.stop()
+ }
+
+ sc = new SparkContext("local-cluster[4,1,1024]", "test", conf)
+ sc
+ }
+
+ override def afterAll(): Unit = {
+ if (sc != null) {
+ sc.stop()
+ }
+ }
+
+}
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index ac25bcef54349..a7eed4b6a8b88 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -18,9 +18,13 @@
package org.apache.spark.serializer
import java.io.{ByteArrayInputStream, ByteArrayOutputStream, FileInputStream, FileOutputStream}
+import java.nio.ByteBuffer
+import java.util.concurrent.Executors
import scala.collection.JavaConverters._
import scala.collection.mutable
+import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration._
import scala.reflect.ClassTag
import com.esotericsoftware.kryo.{Kryo, KryoException}
@@ -31,7 +35,7 @@ import org.apache.spark.{SharedSparkContext, SparkConf, SparkFunSuite}
import org.apache.spark.scheduler.HighlyCompressedMapStatus
import org.apache.spark.serializer.KryoTest._
import org.apache.spark.storage.BlockManagerId
-import org.apache.spark.util.Utils
+import org.apache.spark.util.{ThreadUtils, Utils}
class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
@@ -199,7 +203,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
def check[T: ClassTag](t: T) {
assert(ser.deserialize[T](ser.serialize(t)) === t)
// Check that very long ranges don't get written one element at a time
- assert(ser.serialize(t).limit() < 100)
+ assert(ser.serialize(t).limit() < 200)
}
check(1 to 1000000)
check(1 to 1000000 by 2)
@@ -209,10 +213,10 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
check(1L to 1000000L by 2L)
check(1L until 1000000L)
check(1L until 1000000L by 2L)
- check(1.0 to 1000000.0 by 1.0)
- check(1.0 to 1000000.0 by 2.0)
- check(1.0 until 1000000.0 by 1.0)
- check(1.0 until 1000000.0 by 2.0)
+ check(Range.BigDecimal.inclusive(1, 1000000, 1))
+ check(Range.BigDecimal.inclusive(1, 1000000, 2))
+ check(Range.BigDecimal(1, 1000000, 1))
+ check(Range.BigDecimal(1, 1000000, 2))
}
test("asJavaIterable") {
@@ -308,7 +312,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
val conf = new SparkConf(false)
conf.set("spark.kryo.registrator", "this.class.does.not.exist")
- val thrown = intercept[SparkException](new KryoSerializer(conf).newInstance())
+ val thrown = intercept[SparkException](new KryoSerializer(conf).newInstance().serialize(1))
assert(thrown.getMessage.contains("Failed to register classes with Kryo"))
}
@@ -431,9 +435,11 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
ser.deserialize[HashMap[Int, List[String]]](serializedMap)
}
- private def testSerializerInstanceReuse(autoReset: Boolean, referenceTracking: Boolean): Unit = {
+ private def testSerializerInstanceReuse(
+ autoReset: Boolean, referenceTracking: Boolean, usePool: Boolean): Unit = {
val conf = new SparkConf(loadDefaults = false)
.set("spark.kryo.referenceTracking", referenceTracking.toString)
+ .set("spark.kryo.pool", usePool.toString)
if (!autoReset) {
conf.set("spark.kryo.registrator", classOf[RegistratorWithoutAutoReset].getName)
}
@@ -456,9 +462,58 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
// Regression test for SPARK-7766, an issue where disabling auto-reset and enabling
// reference-tracking would lead to corrupted output when serializer instances are re-used
- for (referenceTracking <- Set(true, false); autoReset <- Set(true, false)) {
- test(s"instance reuse with autoReset = $autoReset, referenceTracking = $referenceTracking") {
- testSerializerInstanceReuse(autoReset = autoReset, referenceTracking = referenceTracking)
+ for {
+ referenceTracking <- Seq(true, false)
+ autoReset <- Seq(true, false)
+ usePool <- Seq(true, false)
+ } {
+ test(s"instance reuse with autoReset = $autoReset, referenceTracking = $referenceTracking" +
+ s", usePool = $usePool") {
+ testSerializerInstanceReuse(
+ autoReset, referenceTracking, usePool)
+ }
+ }
+
+ test("SPARK-25839 KryoPool implementation works correctly in multi-threaded environment") {
+ implicit val executionContext: ExecutionContext = ExecutionContext.fromExecutor(
+ Executors.newFixedThreadPool(4))
+
+ val ser = new KryoSerializer(conf.clone.set("spark.kryo.pool", "true"))
+
+ val tests = mutable.ListBuffer[Future[Boolean]]()
+
+ def check[T: ClassTag](t: T) {
+ tests += Future {
+ val serializerInstance = ser.newInstance()
+ serializerInstance.deserialize[T](serializerInstance.serialize(t)) === t
+ }
+ }
+
+ check((1, 3))
+ check(Array((1, 3)))
+ check(List((1, 3)))
+ check(List[Int]())
+ check(List[Int](1, 2, 3))
+ check(List[String]())
+ check(List[String]("x", "y", "z"))
+ check(None)
+ check(Some(1))
+ check(Some("hi"))
+ check(1 -> 1)
+ check(mutable.ArrayBuffer(1, 2, 3))
+ check(mutable.ArrayBuffer("1", "2", "3"))
+ check(mutable.Map())
+ check(mutable.Map(1 -> "one", 2 -> "two"))
+ check(mutable.Map("one" -> 1, "two" -> 2))
+ check(mutable.HashMap(1 -> "one", 2 -> "two"))
+ check(mutable.HashMap("one" -> 1, "two" -> 2))
+ check(List(Some(mutable.HashMap(1 -> 1, 2 -> 2)), None, Some(mutable.HashMap(3 -> 4))))
+ check(List(
+ mutable.HashMap("one" -> 1, "two" -> 2),
+ mutable.HashMap(1 -> "one", 2 -> "two", 3 -> "three")))
+
+ tests.foreach { f =>
+ assert(ThreadUtils.awaitResult(f, 10.seconds))
}
}
}
@@ -497,6 +552,17 @@ class KryoSerializerAutoResetDisabledSuite extends SparkFunSuite with SharedSpar
deserializationStream.close()
assert(serInstance.deserialize[Any](helloHello) === ((hello, hello)))
}
+
+ test("SPARK-25786: ByteBuffer.array -- UnsupportedOperationException") {
+ val serInstance = new KryoSerializer(conf).newInstance().asInstanceOf[KryoSerializerInstance]
+ val obj = "UnsupportedOperationException"
+ val serObj = serInstance.serialize(obj)
+ val byteBuffer = ByteBuffer.allocateDirect(serObj.array().length)
+ byteBuffer.put(serObj.array())
+ byteBuffer.flip()
+ assert(serInstance.deserialize[Any](serObj) === (obj))
+ assert(serInstance.deserialize[Any](byteBuffer) === (obj))
+ }
}
class ClassLoaderTestingObject
diff --git a/core/src/test/scala/org/apache/spark/shuffle/BlockStoreShuffleReaderSuite.scala b/core/src/test/scala/org/apache/spark/shuffle/BlockStoreShuffleReaderSuite.scala
index 2d8a83c6fabed..eb97d5a1e5074 100644
--- a/core/src/test/scala/org/apache/spark/shuffle/BlockStoreShuffleReaderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/shuffle/BlockStoreShuffleReaderSuite.scala
@@ -126,11 +126,14 @@ class BlockStoreShuffleReaderSuite extends SparkFunSuite with LocalSparkContext
.set("spark.shuffle.compress", "false")
.set("spark.shuffle.spill.compress", "false"))
+ val taskContext = TaskContext.empty()
+ val metrics = taskContext.taskMetrics.createTempShuffleReadMetrics()
val shuffleReader = new BlockStoreShuffleReader(
shuffleHandle,
reduceId,
reduceId + 1,
- TaskContext.empty(),
+ taskContext,
+ metrics,
serializerManager,
blockManager,
mapOutputTracker)
diff --git a/core/src/test/scala/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriterSuite.scala b/core/src/test/scala/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriterSuite.scala
index 85ccb33471048..4467c3241a947 100644
--- a/core/src/test/scala/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriterSuite.scala
@@ -136,8 +136,8 @@ class BypassMergeSortShuffleWriterSuite extends SparkFunSuite with BeforeAndAfte
blockResolver,
shuffleHandle,
0, // MapId
- taskContext,
- conf
+ conf,
+ taskContext.taskMetrics().shuffleWriteMetrics
)
writer.write(Iterator.empty)
writer.stop( /* success = */ true)
@@ -160,8 +160,8 @@ class BypassMergeSortShuffleWriterSuite extends SparkFunSuite with BeforeAndAfte
blockResolver,
shuffleHandle,
0, // MapId
- taskContext,
- conf
+ conf,
+ taskContext.taskMetrics().shuffleWriteMetrics
)
writer.write(records)
writer.stop( /* success = */ true)
@@ -195,8 +195,8 @@ class BypassMergeSortShuffleWriterSuite extends SparkFunSuite with BeforeAndAfte
blockResolver,
shuffleHandle,
0, // MapId
- taskContext,
- conf
+ conf,
+ taskContext.taskMetrics().shuffleWriteMetrics
)
intercept[SparkException] {
@@ -217,8 +217,8 @@ class BypassMergeSortShuffleWriterSuite extends SparkFunSuite with BeforeAndAfte
blockResolver,
shuffleHandle,
0, // MapId
- taskContext,
- conf
+ conf,
+ taskContext.taskMetrics().shuffleWriteMetrics
)
intercept[SparkException] {
writer.write((0 until 100000).iterator.map(i => {
diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
index bfd73069fbff8..7860a0df4bb2d 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
@@ -18,8 +18,7 @@
package org.apache.spark.status
import java.io.File
-import java.lang.{Integer => JInteger, Long => JLong}
-import java.util.{Arrays, Date, Properties}
+import java.util.{Date, Properties}
import scala.collection.JavaConverters._
import scala.collection.immutable.Map
@@ -1171,12 +1170,12 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
// Stop task 2 before task 1
time += 1
tasks(1).markFinished(TaskState.FINISHED, time)
- listener.onTaskEnd(
- SparkListenerTaskEnd(stage1.stageId, stage1.attemptId, "taskType", Success, tasks(1), null))
+ listener.onTaskEnd(SparkListenerTaskEnd(
+ stage1.stageId, stage1.attemptNumber, "taskType", Success, tasks(1), null))
time += 1
tasks(0).markFinished(TaskState.FINISHED, time)
- listener.onTaskEnd(
- SparkListenerTaskEnd(stage1.stageId, stage1.attemptId, "taskType", Success, tasks(0), null))
+ listener.onTaskEnd(SparkListenerTaskEnd(
+ stage1.stageId, stage1.attemptNumber, "taskType", Success, tasks(0), null))
// Start task 3 and task 2 should be evicted.
listener.onTaskStart(SparkListenerTaskStart(stage1.stageId, stage1.attemptNumber, tasks(2)))
@@ -1241,8 +1240,8 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
// Task 1 Finished
time += 1
tasks(0).markFinished(TaskState.FINISHED, time)
- listener.onTaskEnd(
- SparkListenerTaskEnd(stage1.stageId, stage1.attemptId, "taskType", Success, tasks(0), null))
+ listener.onTaskEnd(SparkListenerTaskEnd(
+ stage1.stageId, stage1.attemptNumber, "taskType", Success, tasks(0), null))
// Stage 1 Completed
stage1.failureReason = Some("Failed")
@@ -1256,7 +1255,7 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
time += 1
tasks(1).markFinished(TaskState.FINISHED, time)
listener.onTaskEnd(
- SparkListenerTaskEnd(stage1.stageId, stage1.attemptId, "taskType",
+ SparkListenerTaskEnd(stage1.stageId, stage1.attemptNumber, "taskType",
TaskKilled(reason = "Killed"), tasks(1), null))
// Ensure killed task metrics are updated
@@ -1274,6 +1273,51 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
assert(allJobs.head.numFailedStages == 1)
}
+ test("SPARK-25451: total tasks in the executor summary should match total stage tasks") {
+ val testConf = conf.clone.set(LIVE_ENTITY_UPDATE_PERIOD, Long.MaxValue)
+
+ val listener = new AppStatusListener(store, testConf, true)
+
+ val stage = new StageInfo(1, 0, "stage", 4, Nil, Nil, "details")
+ listener.onJobStart(SparkListenerJobStart(1, time, Seq(stage), null))
+ listener.onStageSubmitted(SparkListenerStageSubmitted(stage, new Properties()))
+
+ val tasks = createTasks(4, Array("1", "2"))
+ tasks.foreach { task =>
+ listener.onTaskStart(SparkListenerTaskStart(stage.stageId, stage.attemptNumber, task))
+ }
+
+ time += 1
+ tasks(0).markFinished(TaskState.FINISHED, time)
+ listener.onTaskEnd(SparkListenerTaskEnd(stage.stageId, stage.attemptNumber, "taskType",
+ Success, tasks(0), null))
+ time += 1
+ tasks(1).markFinished(TaskState.FINISHED, time)
+ listener.onTaskEnd(SparkListenerTaskEnd(stage.stageId, stage.attemptNumber, "taskType",
+ Success, tasks(1), null))
+
+ stage.failureReason = Some("Failed")
+ listener.onStageCompleted(SparkListenerStageCompleted(stage))
+ time += 1
+ listener.onJobEnd(SparkListenerJobEnd(1, time, JobFailed(new RuntimeException("Bad Executor"))))
+
+ time += 1
+ tasks(2).markFinished(TaskState.FAILED, time)
+ listener.onTaskEnd(SparkListenerTaskEnd(stage.stageId, stage.attemptNumber, "taskType",
+ ExecutorLostFailure("1", true, Some("Lost executor")), tasks(2), null))
+ time += 1
+ tasks(3).markFinished(TaskState.FAILED, time)
+ listener.onTaskEnd(SparkListenerTaskEnd(stage.stageId, stage.attemptNumber, "taskType",
+ ExecutorLostFailure("2", true, Some("Lost executor")), tasks(3), null))
+
+ val esummary = store.view(classOf[ExecutorStageSummaryWrapper]).asScala.map(_.info)
+ esummary.foreach { execSummary =>
+ assert(execSummary.failedTasks === 1)
+ assert(execSummary.succeededTasks === 1)
+ assert(execSummary.killedTasks === 0)
+ }
+ }
+
test("driver logs") {
val listener = new AppStatusListener(store, conf, true)
diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala
index 9e74e86ad54b9..a01b24d323d28 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala
@@ -52,7 +52,10 @@ class AppStatusUtilsSuite extends SparkFunSuite {
inputMetrics = null,
outputMetrics = null,
shuffleReadMetrics = null,
- shuffleWriteMetrics = null)))
+ shuffleWriteMetrics = null)),
+ executorLogs = null,
+ schedulerDelay = 0L,
+ gettingResultTime = 0L)
assert(AppStatusUtils.schedulerDelay(runningTask) === 0L)
val finishedTask = new TaskData(
@@ -83,7 +86,10 @@ class AppStatusUtilsSuite extends SparkFunSuite {
inputMetrics = null,
outputMetrics = null,
shuffleReadMetrics = null,
- shuffleWriteMetrics = null)))
+ shuffleWriteMetrics = null)),
+ executorLogs = null,
+ schedulerDelay = 0L,
+ gettingResultTime = 0L)
assert(AppStatusUtils.schedulerDelay(finishedTask) === 3L)
}
}
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
index 32d6e8b94e1a2..cf00c1c3aad39 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
@@ -574,7 +574,7 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
"list1",
StorageLevel.MEMORY_ONLY,
ClassTag.Any,
- () => throw new AssertionError("attempted to compute locally")).isLeft)
+ () => fail("attempted to compute locally")).isLeft)
}
test("in-memory LRU storage") {
diff --git a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
index b268195e09a5b..01ee9ef0825f8 100644
--- a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
@@ -102,8 +102,10 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
(remoteBmId, remoteBlocks.keys.map(blockId => (blockId, 1.asInstanceOf[Long])).toSeq)
).toIterator
+ val taskContext = TaskContext.empty()
+ val metrics = taskContext.taskMetrics.createTempShuffleReadMetrics()
val iterator = new ShuffleBlockFetcherIterator(
- TaskContext.empty(),
+ taskContext,
transfer,
blockManager,
blocksByAddress,
@@ -112,7 +114,8 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
Int.MaxValue,
Int.MaxValue,
Int.MaxValue,
- true)
+ true,
+ metrics)
// 3 local blocks fetched in initialization
verify(blockManager, times(3)).getBlockData(any())
@@ -190,7 +193,8 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
Int.MaxValue,
Int.MaxValue,
Int.MaxValue,
- true)
+ true,
+ taskContext.taskMetrics.createTempShuffleReadMetrics())
verify(blocks(ShuffleBlockId(0, 0, 0)), times(0)).release()
iterator.next()._2.close() // close() first block's input stream
@@ -258,7 +262,8 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
Int.MaxValue,
Int.MaxValue,
Int.MaxValue,
- true)
+ true,
+ taskContext.taskMetrics.createTempShuffleReadMetrics())
// Continue only after the mock calls onBlockFetchFailure
sem.acquire()
@@ -328,7 +333,8 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
Int.MaxValue,
Int.MaxValue,
Int.MaxValue,
- true)
+ true,
+ taskContext.taskMetrics.createTempShuffleReadMetrics())
// Continue only after the mock calls onBlockFetchFailure
sem.acquire()
@@ -392,7 +398,8 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
Int.MaxValue,
Int.MaxValue,
Int.MaxValue,
- true)
+ true,
+ taskContext.taskMetrics.createTempShuffleReadMetrics())
// Blocks should be returned without exceptions.
assert(Set(iterator.next()._1, iterator.next()._1) ===
Set(ShuffleBlockId(0, 0, 0), ShuffleBlockId(0, 1, 0)))
@@ -446,7 +453,8 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
Int.MaxValue,
Int.MaxValue,
Int.MaxValue,
- false)
+ false,
+ taskContext.taskMetrics.createTempShuffleReadMetrics())
// Continue only after the mock calls onBlockFetchFailure
sem.acquire()
@@ -496,8 +504,9 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
// Set `maxBytesInFlight` and `maxReqsInFlight` to `Int.MaxValue`, so that during the
// construction of `ShuffleBlockFetcherIterator`, all requests to fetch remote shuffle blocks
// are issued. The `maxReqSizeShuffleToMem` is hard-coded as 200 here.
+ val taskContext = TaskContext.empty()
new ShuffleBlockFetcherIterator(
- TaskContext.empty(),
+ taskContext,
transfer,
blockManager,
blocksByAddress,
@@ -506,7 +515,8 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
maxReqsInFlight = Int.MaxValue,
maxBlocksInFlightPerAddress = Int.MaxValue,
maxReqSizeShuffleToMem = 200,
- detectCorrupt = true)
+ detectCorrupt = true,
+ taskContext.taskMetrics.createTempShuffleReadMetrics())
}
val blocksByAddress1 = Seq[(BlockManagerId, Seq[(BlockId, Long)])](
@@ -552,7 +562,8 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
Int.MaxValue,
Int.MaxValue,
Int.MaxValue,
- true)
+ true,
+ taskContext.taskMetrics.createTempShuffleReadMetrics())
// All blocks fetched return zero length and should trigger a receive-side error:
val e = intercept[FetchFailedException] { iterator.next() }
diff --git a/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala b/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
index 2945c3ee0a9d9..5e976ae4e91da 100644
--- a/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
@@ -96,18 +96,6 @@ class StagePageSuite extends SparkFunSuite with LocalSparkContext {
}
}
- test("peak execution memory should displayed") {
- val html = renderStagePage().toString().toLowerCase(Locale.ROOT)
- val targetString = "peak execution memory"
- assert(html.contains(targetString))
- }
-
- test("SPARK-10543: peak execution memory should be per-task rather than cumulative") {
- val html = renderStagePage().toString().toLowerCase(Locale.ROOT)
- // verify min/25/50/75/max show task value not cumulative values
- assert(html.contains(s"| $peakExecutionMemory.0 b | " * 5))
- }
-
/**
* Render a stage page started with the given conf and return the HTML.
* This also runs a dummy stage to populate the page with useful content.
diff --git a/core/src/test/scala/org/apache/spark/ui/storage/StoragePageSuite.scala b/core/src/test/scala/org/apache/spark/ui/storage/StoragePageSuite.scala
index cdc7f541b9552..06f01a60868f9 100644
--- a/core/src/test/scala/org/apache/spark/ui/storage/StoragePageSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/storage/StoragePageSuite.scala
@@ -81,19 +81,19 @@ class StoragePageSuite extends SparkFunSuite {
Seq("1", "rdd1", "Memory Deserialized 1x Replicated", "10", "100%", "100.0 B", "0.0 B"))
// Check the url
assert(((xmlNodes \\ "tr")(0) \\ "td" \ "a")(0).attribute("href").map(_.text) ===
- Some("http://localhost:4040/storage/rdd?id=1"))
+ Some("http://localhost:4040/storage/rdd/?id=1"))
assert(((xmlNodes \\ "tr")(1) \\ "td").map(_.text.trim) ===
Seq("2", "rdd2", "Disk Serialized 1x Replicated", "5", "50%", "0.0 B", "200.0 B"))
// Check the url
assert(((xmlNodes \\ "tr")(1) \\ "td" \ "a")(0).attribute("href").map(_.text) ===
- Some("http://localhost:4040/storage/rdd?id=2"))
+ Some("http://localhost:4040/storage/rdd/?id=2"))
assert(((xmlNodes \\ "tr")(2) \\ "td").map(_.text.trim) ===
Seq("3", "rdd3", "Disk Memory Serialized 1x Replicated", "10", "100%", "400.0 B", "500.0 B"))
// Check the url
assert(((xmlNodes \\ "tr")(2) \\ "td" \ "a")(0).attribute("href").map(_.text) ===
- Some("http://localhost:4040/storage/rdd?id=3"))
+ Some("http://localhost:4040/storage/rdd/?id=3"))
}
test("empty rddTable") {
diff --git a/core/src/test/scala/org/apache/spark/util/AccumulatorV2Suite.scala b/core/src/test/scala/org/apache/spark/util/AccumulatorV2Suite.scala
index 621399af731f7..172bebbfec61d 100644
--- a/core/src/test/scala/org/apache/spark/util/AccumulatorV2Suite.scala
+++ b/core/src/test/scala/org/apache/spark/util/AccumulatorV2Suite.scala
@@ -40,7 +40,7 @@ class AccumulatorV2Suite extends SparkFunSuite {
assert(acc.avg == 0.5)
// Also test add using non-specialized add function
- acc.add(new java.lang.Long(2))
+ acc.add(java.lang.Long.valueOf(2))
assert(acc.count == 3)
assert(acc.sum == 3)
assert(acc.avg == 1.0)
@@ -73,7 +73,7 @@ class AccumulatorV2Suite extends SparkFunSuite {
assert(acc.avg == 0.5)
// Also test add using non-specialized add function
- acc.add(new java.lang.Double(2.0))
+ acc.add(java.lang.Double.valueOf(2.0))
assert(acc.count == 3)
assert(acc.sum == 3.0)
assert(acc.avg == 1.0)
@@ -96,7 +96,7 @@ class AccumulatorV2Suite extends SparkFunSuite {
assert(acc.value.contains(0.0))
assert(!acc.isZero)
- acc.add(new java.lang.Double(1.0))
+ acc.add(java.lang.Double.valueOf(1.0))
val acc2 = acc.copyAndReset()
assert(acc2.value.isEmpty)
diff --git a/core/src/test/scala/org/apache/spark/util/CompletionIteratorSuite.scala b/core/src/test/scala/org/apache/spark/util/CompletionIteratorSuite.scala
index 688fcd9f9aaba..29421f7aa9e36 100644
--- a/core/src/test/scala/org/apache/spark/util/CompletionIteratorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/CompletionIteratorSuite.scala
@@ -17,6 +17,9 @@
package org.apache.spark.util
+import java.lang.ref.PhantomReference
+import java.lang.ref.ReferenceQueue
+
import org.apache.spark.SparkFunSuite
class CompletionIteratorSuite extends SparkFunSuite {
@@ -44,4 +47,23 @@ class CompletionIteratorSuite extends SparkFunSuite {
assert(!completionIter.hasNext)
assert(numTimesCompleted === 1)
}
+ test("reference to sub iterator should not be available after completion") {
+ var sub = Iterator(1, 2, 3)
+
+ val refQueue = new ReferenceQueue[Iterator[Int]]
+ val ref = new PhantomReference[Iterator[Int]](sub, refQueue)
+
+ val iter = CompletionIterator[Int, Iterator[Int]](sub, {})
+ sub = null
+ iter.toArray
+
+ for (_ <- 1 to 100 if !ref.isEnqueued) {
+ System.gc()
+ if (!ref.isEnqueued) {
+ Thread.sleep(10)
+ }
+ }
+ assert(ref.isEnqueued)
+ assert(refQueue.poll() === ref)
+ }
}
diff --git a/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala b/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala
index f6ac89fc2742a..8d844bd08771c 100644
--- a/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala
@@ -22,7 +22,6 @@ import java.net.URLClassLoader
import scala.collection.JavaConverters._
import org.scalatest.Matchers
-import org.scalatest.Matchers._
import org.apache.spark.{SparkContext, SparkException, SparkFunSuite, TestUtils}
@@ -46,10 +45,10 @@ class MutableURLClassLoaderSuite extends SparkFunSuite with Matchers {
test("child first") {
val parentLoader = new URLClassLoader(urls2, null)
val classLoader = new ChildFirstURLClassLoader(urls, parentLoader)
- val fakeClass = classLoader.loadClass("FakeClass2").newInstance()
+ val fakeClass = classLoader.loadClass("FakeClass2").getConstructor().newInstance()
val fakeClassVersion = fakeClass.toString
assert(fakeClassVersion === "1")
- val fakeClass2 = classLoader.loadClass("FakeClass2").newInstance()
+ val fakeClass2 = classLoader.loadClass("FakeClass2").getConstructor().newInstance()
assert(fakeClass.getClass === fakeClass2.getClass)
classLoader.close()
parentLoader.close()
@@ -58,10 +57,10 @@ class MutableURLClassLoaderSuite extends SparkFunSuite with Matchers {
test("parent first") {
val parentLoader = new URLClassLoader(urls2, null)
val classLoader = new MutableURLClassLoader(urls, parentLoader)
- val fakeClass = classLoader.loadClass("FakeClass1").newInstance()
+ val fakeClass = classLoader.loadClass("FakeClass1").getConstructor().newInstance()
val fakeClassVersion = fakeClass.toString
assert(fakeClassVersion === "2")
- val fakeClass2 = classLoader.loadClass("FakeClass1").newInstance()
+ val fakeClass2 = classLoader.loadClass("FakeClass1").getConstructor().newInstance()
assert(fakeClass.getClass === fakeClass2.getClass)
classLoader.close()
parentLoader.close()
@@ -70,7 +69,7 @@ class MutableURLClassLoaderSuite extends SparkFunSuite with Matchers {
test("child first can fall back") {
val parentLoader = new URLClassLoader(urls2, null)
val classLoader = new ChildFirstURLClassLoader(urls, parentLoader)
- val fakeClass = classLoader.loadClass("FakeClass3").newInstance()
+ val fakeClass = classLoader.loadClass("FakeClass3").getConstructor().newInstance()
val fakeClassVersion = fakeClass.toString
assert(fakeClassVersion === "2")
classLoader.close()
@@ -81,7 +80,7 @@ class MutableURLClassLoaderSuite extends SparkFunSuite with Matchers {
val parentLoader = new URLClassLoader(urls2, null)
val classLoader = new ChildFirstURLClassLoader(urls, parentLoader)
intercept[java.lang.ClassNotFoundException] {
- classLoader.loadClass("FakeClassDoesNotExist").newInstance()
+ classLoader.loadClass("FakeClassDoesNotExist").getConstructor().newInstance()
}
classLoader.close()
parentLoader.close()
@@ -137,7 +136,7 @@ class MutableURLClassLoaderSuite extends SparkFunSuite with Matchers {
sc.makeRDD(1 to 5, 2).mapPartitions { x =>
val loader = Thread.currentThread().getContextClassLoader
// scalastyle:off classforname
- Class.forName(className, true, loader).newInstance()
+ Class.forName(className, true, loader).getConstructor().newInstance()
// scalastyle:on classforname
Seq().iterator
}.count()
diff --git a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
index 2695295d451d5..63f9f82adf3e0 100644
--- a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
@@ -92,14 +92,14 @@ class SizeEstimatorSuite
}
test("primitive wrapper objects") {
- assertResult(16)(SizeEstimator.estimate(new java.lang.Boolean(true)))
- assertResult(16)(SizeEstimator.estimate(new java.lang.Byte("1")))
- assertResult(16)(SizeEstimator.estimate(new java.lang.Character('1')))
- assertResult(16)(SizeEstimator.estimate(new java.lang.Short("1")))
- assertResult(16)(SizeEstimator.estimate(new java.lang.Integer(1)))
- assertResult(24)(SizeEstimator.estimate(new java.lang.Long(1)))
- assertResult(16)(SizeEstimator.estimate(new java.lang.Float(1.0)))
- assertResult(24)(SizeEstimator.estimate(new java.lang.Double(1.0d)))
+ assertResult(16)(SizeEstimator.estimate(java.lang.Boolean.TRUE))
+ assertResult(16)(SizeEstimator.estimate(java.lang.Byte.valueOf("1")))
+ assertResult(16)(SizeEstimator.estimate(java.lang.Character.valueOf('1')))
+ assertResult(16)(SizeEstimator.estimate(java.lang.Short.valueOf("1")))
+ assertResult(16)(SizeEstimator.estimate(java.lang.Integer.valueOf(1)))
+ assertResult(24)(SizeEstimator.estimate(java.lang.Long.valueOf(1)))
+ assertResult(16)(SizeEstimator.estimate(java.lang.Float.valueOf(1.0f)))
+ assertResult(24)(SizeEstimator.estimate(java.lang.Double.valueOf(1.0)))
}
test("class field blocks rounding") {
@@ -202,14 +202,14 @@ class SizeEstimatorSuite
assertResult(72)(SizeEstimator.estimate(DummyString("abcdefgh")))
// primitive wrapper classes
- assertResult(24)(SizeEstimator.estimate(new java.lang.Boolean(true)))
- assertResult(24)(SizeEstimator.estimate(new java.lang.Byte("1")))
- assertResult(24)(SizeEstimator.estimate(new java.lang.Character('1')))
- assertResult(24)(SizeEstimator.estimate(new java.lang.Short("1")))
- assertResult(24)(SizeEstimator.estimate(new java.lang.Integer(1)))
- assertResult(24)(SizeEstimator.estimate(new java.lang.Long(1)))
- assertResult(24)(SizeEstimator.estimate(new java.lang.Float(1.0)))
- assertResult(24)(SizeEstimator.estimate(new java.lang.Double(1.0d)))
+ assertResult(24)(SizeEstimator.estimate(java.lang.Boolean.TRUE))
+ assertResult(24)(SizeEstimator.estimate(java.lang.Byte.valueOf("1")))
+ assertResult(24)(SizeEstimator.estimate(java.lang.Character.valueOf('1')))
+ assertResult(24)(SizeEstimator.estimate(java.lang.Short.valueOf("1")))
+ assertResult(24)(SizeEstimator.estimate(java.lang.Integer.valueOf(1)))
+ assertResult(24)(SizeEstimator.estimate(java.lang.Long.valueOf(1)))
+ assertResult(24)(SizeEstimator.estimate(java.lang.Float.valueOf(1.0f)))
+ assertResult(24)(SizeEstimator.estimate(java.lang.Double.valueOf(1.0)))
}
test("class field blocks rounding on 64-bit VM without useCompressedOops") {
diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
index 39f4fba78583f..f5e912b50d1ab 100644
--- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
@@ -33,7 +33,7 @@ import scala.util.Random
import com.google.common.io.Files
import org.apache.commons.io.IOUtils
-import org.apache.commons.lang3.SystemUtils
+import org.apache.commons.lang3.{JavaVersion, SystemUtils}
import org.apache.commons.math3.stat.inference.ChiSquareTest
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
@@ -45,14 +45,6 @@ import org.apache.spark.scheduler.SparkListener
class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
- test("truncatedString") {
- assert(Utils.truncatedString(Nil, "[", ", ", "]", 2) == "[]")
- assert(Utils.truncatedString(Seq(1, 2), "[", ", ", "]", 2) == "[1, 2]")
- assert(Utils.truncatedString(Seq(1, 2, 3), "[", ", ", "]", 2) == "[1, ... 2 more fields]")
- assert(Utils.truncatedString(Seq(1, 2, 3), "[", ", ", "]", -5) == "[, ... 3 more fields]")
- assert(Utils.truncatedString(Seq(1, 2, 3), ", ") == "1, 2, 3")
- }
-
test("timeConversion") {
// Test -1
assert(Utils.timeStringAsSeconds("-1") === -1)
@@ -932,10 +924,7 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
signal(pid, "SIGKILL")
}
- val versionParts = System.getProperty("java.version").split("[+.\\-]+", 3)
- var majorVersion = versionParts(0).toInt
- if (majorVersion == 1) majorVersion = versionParts(1).toInt
- if (majorVersion >= 8) {
+ if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_1_8)) {
// We'll make sure that forcibly terminating a process works by
// creating a very misbehaving process. It ignores SIGTERM and has been SIGSTOPed. On
// older versions of java, this will *not* terminate.
diff --git a/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
index b36d6be231d39..56623ebea1651 100644
--- a/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
@@ -73,4 +73,29 @@ class VersionUtilsSuite extends SparkFunSuite {
}
}
}
+
+ test("Return short version number") {
+ assert(shortVersion("3.0.0") === "3.0.0")
+ assert(shortVersion("3.0.0-SNAPSHOT") === "3.0.0")
+ withClue("shortVersion parsing should fail for missing maintenance version number") {
+ intercept[IllegalArgumentException] {
+ shortVersion("3.0")
+ }
+ }
+ withClue("shortVersion parsing should fail for invalid major version number") {
+ intercept[IllegalArgumentException] {
+ shortVersion("x.0.0")
+ }
+ }
+ withClue("shortVersion parsing should fail for invalid minor version number") {
+ intercept[IllegalArgumentException] {
+ shortVersion("3.x.0")
+ }
+ }
+ withClue("shortVersion parsing should fail for invalid maintenance version number") {
+ intercept[IllegalArgumentException] {
+ shortVersion("3.0.x")
+ }
+ }
+ }
}
diff --git a/core/src/test/scala/org/apache/spark/util/collection/ExternalAppendOnlyMapSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/ExternalAppendOnlyMapSuite.scala
index cd25265784136..35fba1a3b73c6 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/ExternalAppendOnlyMapSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/ExternalAppendOnlyMapSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.util.collection
import scala.collection.mutable.ArrayBuffer
import scala.concurrent.duration._
+import scala.language.postfixOps
import scala.ref.WeakReference
import org.scalatest.Matchers
diff --git a/core/src/test/scala/org/apache/spark/util/collection/SorterSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/SorterSuite.scala
index 65bf857e22c02..46a05e2ba798b 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/SorterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/SorterSuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.util.collection
-import java.lang.{Float => JFloat, Integer => JInteger}
+import java.lang.{Float => JFloat}
import java.util.{Arrays, Comparator}
import org.apache.spark.SparkFunSuite
@@ -48,7 +48,7 @@ class SorterSuite extends SparkFunSuite with Logging {
// alternate. Keys are random doubles, values are ordinals from 0 to length.
val keys = Array.tabulate[Double](5000) { i => rand.nextDouble() }
val keyValueArray = Array.tabulate[Number](10000) { i =>
- if (i % 2 == 0) keys(i / 2) else new Integer(i / 2)
+ if (i % 2 == 0) keys(i / 2) else Integer.valueOf(i / 2)
}
// Map from generated keys to values, to verify correctness later
@@ -112,7 +112,7 @@ class SorterSuite extends SparkFunSuite with Logging {
// Test our key-value pairs where each element is a Tuple2[Float, Integer].
val kvTuples = Array.tabulate(numElements) { i =>
- (new JFloat(rand.nextFloat()), new JInteger(i))
+ (JFloat.valueOf(rand.nextFloat()), Integer.valueOf(i))
}
val kvTupleArray = new Array[AnyRef](numElements)
@@ -167,23 +167,23 @@ class SorterSuite extends SparkFunSuite with Logging {
val ints = Array.fill(numElements)(rand.nextInt())
val intObjects = {
- val data = new Array[JInteger](numElements)
+ val data = new Array[Integer](numElements)
var i = 0
while (i < numElements) {
- data(i) = new JInteger(ints(i))
+ data(i) = Integer.valueOf(ints(i))
i += 1
}
data
}
- val intObjectArray = new Array[JInteger](numElements)
+ val intObjectArray = new Array[Integer](numElements)
val prepareIntObjectArray = () => {
System.arraycopy(intObjects, 0, intObjectArray, 0, numElements)
}
runExperiment("Java Arrays.sort() on non-primitive int array")({
- Arrays.sort(intObjectArray, new Comparator[JInteger] {
- override def compare(x: JInteger, y: JInteger): Int = x.compareTo(y)
+ Arrays.sort(intObjectArray, new Comparator[Integer] {
+ override def compare(x: Integer, y: Integer): Int = x.compareTo(y)
})
}, prepareIntObjectArray)
diff --git a/core/src/test/scala/org/apache/spark/util/collection/unsafe/sort/RadixSortSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/unsafe/sort/RadixSortSuite.scala
index d5956ea32096a..d570630c1a095 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/unsafe/sort/RadixSortSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/unsafe/sort/RadixSortSuite.scala
@@ -78,7 +78,7 @@ class RadixSortSuite extends SparkFunSuite with Logging {
private def generateTestData(size: Long, rand: => Long): (Array[JLong], LongArray) = {
val ref = Array.tabulate[Long](Ints.checkedCast(size)) { i => rand }
val extended = ref ++ Array.fill[Long](Ints.checkedCast(size))(0)
- (ref.map(i => new JLong(i)), new LongArray(MemoryBlock.fromLongArray(extended)))
+ (ref.map(i => JLong.valueOf(i)), new LongArray(MemoryBlock.fromLongArray(extended)))
}
private def generateKeyPrefixTestData(size: Long, rand: => Long): (LongArray, LongArray) = {
diff --git a/core/src/test/scala/org/apache/spark/util/logging/DriverLoggerSuite.scala b/core/src/test/scala/org/apache/spark/util/logging/DriverLoggerSuite.scala
new file mode 100644
index 0000000000000..973f71cdeb755
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/util/logging/DriverLoggerSuite.scala
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util.logging
+
+import java.io.{BufferedInputStream, File, FileInputStream}
+
+import org.apache.commons.io.FileUtils
+
+import org.apache.spark._
+import org.apache.spark.{SparkContext, SparkFunSuite}
+import org.apache.spark.internal.config._
+import org.apache.spark.launcher.SparkLauncher
+import org.apache.spark.network.util.JavaUtils
+import org.apache.spark.util.Utils
+
+class DriverLoggerSuite extends SparkFunSuite with LocalSparkContext {
+
+ private var rootDfsDir : File = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ rootDfsDir = Utils.createTempDir(namePrefix = "dfs_logs")
+ }
+
+ override def afterAll(): Unit = {
+ super.afterAll()
+ JavaUtils.deleteRecursively(rootDfsDir)
+ }
+
+ test("driver logs are persisted locally and synced to dfs") {
+ val sc = getSparkContext()
+
+ val app_id = sc.applicationId
+ // Run a simple spark application
+ sc.parallelize(1 to 1000).count()
+
+ // Assert driver log file exists
+ val rootDir = Utils.getLocalDir(sc.getConf)
+ val driverLogsDir = FileUtils.getFile(rootDir, DriverLogger.DRIVER_LOG_DIR)
+ assert(driverLogsDir.exists())
+ val files = driverLogsDir.listFiles()
+ assert(files.length === 1)
+ assert(files(0).getName.equals(DriverLogger.DRIVER_LOG_FILE))
+
+ sc.stop()
+ assert(!driverLogsDir.exists())
+ val dfsFile = FileUtils.getFile(sc.getConf.get(DRIVER_LOG_DFS_DIR).get,
+ app_id + DriverLogger.DRIVER_LOG_FILE_SUFFIX)
+ assert(dfsFile.exists())
+ assert(dfsFile.length() > 0)
+ }
+
+ private def getSparkContext(): SparkContext = {
+ val conf = new SparkConf()
+ conf.set(DRIVER_LOG_DFS_DIR, rootDfsDir.getAbsolutePath())
+ conf.set(DRIVER_LOG_PERSISTTODFS, true)
+ conf.set(SparkLauncher.SPARK_MASTER, "local")
+ conf.set(SparkLauncher.DEPLOY_MODE, "client")
+ sc = new SparkContext("local", "DriverLogTest", conf)
+ sc
+ }
+
+}
+
diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1
index 06d9d70af311a..cc68ffb90d875 100644
--- a/dev/appveyor-install-dependencies.ps1
+++ b/dev/appveyor-install-dependencies.ps1
@@ -116,7 +116,7 @@ Pop-Location
# ========================== R
$rVer = "3.5.1"
-$rToolsVer = "3.4.0"
+$rToolsVer = "3.5.1"
InstallR
InstallRtools
diff --git a/dev/checkstyle.xml b/dev/checkstyle.xml
index 53c284888ebb0..e8859c01f2bd8 100644
--- a/dev/checkstyle.xml
+++ b/dev/checkstyle.xml
@@ -71,13 +71,13 @@
If you wish to turn off checking for a section of code, you can put a comment in the source
before and after the section, with the following syntax:
- // checkstyle:off no.XXX (such as checkstyle.off: NoFinalizer)
+ // checkstyle.off: XXX (such as checkstyle.off: NoFinalizer)
... // stuff that breaks the styles
- // checkstyle:on
+ // checkstyle.on: XXX (such as checkstyle.on: NoFinalizer)
-->
-
-
+
+
@@ -180,5 +180,10 @@
+
+
+
+
+
diff --git a/dev/deps/spark-deps-hadoop-palantir b/dev/deps/spark-deps-hadoop-palantir
index 7253e4920b110..0d881c6cdab73 100644
--- a/dev/deps/spark-deps-hadoop-palantir
+++ b/dev/deps/spark-deps-hadoop-palantir
@@ -2,7 +2,7 @@ HikariCP-java7-2.4.12.jar
RoaringBitmap-0.7.16.jar
activation-1.1.1.jar
aircompressor-0.10.jar
-antlr4-runtime-4.7.jar
+antlr4-runtime-4.7.1.jar
aopalliance-1.0.jar
aopalliance-repackaged-2.5.0-b32.jar
apacheds-i18n-2.0.0-M15.jar
@@ -21,10 +21,10 @@ avro-mapred-1.8.2-hadoop2.jar
aws-java-sdk-bundle-1.11.201.jar
azure-keyvault-core-0.8.0.jar
azure-storage-5.4.0.jar
-breeze-macros_2.11-0.13.2.jar
-breeze_2.11-0.13.2.jar
+breeze-macros_2.12-0.13.2.jar
+breeze_2.12-0.13.2.jar
chill-java-0.9.3.jar
-chill_2.11-0.9.3.jar
+chill_2.12-0.9.3.jar
classmate-1.1.0.jar
commons-beanutils-1.9.3.jar
commons-beanutils-core-1.8.0.jar
@@ -32,14 +32,14 @@ commons-cli-1.2.jar
commons-codec-1.11.jar
commons-collections-3.2.2.jar
commons-compiler-3.0.10.jar
-commons-compress-1.8.1.jar
+commons-compress-1.18.jar
commons-configuration-1.6.jar
commons-crypto-1.0.0.jar
commons-digester-1.8.jar
commons-httpclient-3.1.jar
commons-io-2.6.jar
commons-lang-2.6.jar
-commons-lang3-3.8.jar
+commons-lang3-3.8.1.jar
commons-logging-1.2.jar
commons-math3-3.6.1.jar
commons-net-3.6.jar
@@ -105,7 +105,7 @@ jackson-mapper-asl-1.9.13.jar
jackson-module-afterburner-2.9.7.jar
jackson-module-jaxb-annotations-2.9.7.jar
jackson-module-paranamer-2.9.7.jar
-jackson-module-scala_2.11-2.9.7.jar
+jackson-module-scala_2.12-2.9.7.jar
jackson-xc-1.9.13.jar
janino-3.0.10.jar
javassist-3.20.0-GA.jar
@@ -132,10 +132,10 @@ jetty-util-6.1.26.jar
jetty-util-9.4.12.v20180830.jar
joda-time-2.10.jar
json-smart-1.3.1.jar
-json4s-ast_2.11-3.5.3.jar
-json4s-core_2.11-3.5.3.jar
-json4s-jackson_2.11-3.5.3.jar
-json4s-scalap_2.11-3.5.3.jar
+json4s-ast_2.12-3.5.3.jar
+json4s-core_2.12-3.5.3.jar
+json4s-jackson_2.12-3.5.3.jar
+json4s-scalap_2.12-3.5.3.jar
jsp-api-2.1.jar
jsr305-3.0.2.jar
jtransforms-2.4.0.jar
@@ -148,8 +148,8 @@ leveldbjni-all-1.8.jar
log4j-1.2.17.jar
logging-interceptor-3.11.0.jar
lz4-java-1.5.0.jar
-machinist_2.11-0.6.1.jar
-macro-compat_2.11-1.1.1.jar
+machinist_2.12-0.6.1.jar
+macro-compat_2.12-1.1.1.jar
metrics-core-3.2.6.jar
metrics-graphite-3.2.6.jar
metrics-influxdb-1.2.2.jar
@@ -181,18 +181,18 @@ protobuf-java-2.5.0.jar
py4j-0.10.8.1.jar
pyrolite-4.13.jar
safe-logging-1.5.1.jar
-scala-compiler-2.11.12.jar
-scala-library-2.11.12.jar
-scala-parser-combinators_2.11-1.1.0.jar
-scala-reflect-2.11.12.jar
-scala-xml_2.11-1.0.5.jar
-shapeless_2.11-2.3.2.jar
+scala-compiler-2.12.7.jar
+scala-library-2.12.7.jar
+scala-parser-combinators_2.12-1.1.0.jar
+scala-reflect-2.12.7.jar
+scala-xml_2.12-1.0.5.jar
+shapeless_2.12-2.3.2.jar
slf4j-api-1.7.25.jar
slf4j-log4j12-1.7.25.jar
snakeyaml-1.23.jar
snappy-java-1.1.7.2.jar
-spire-macros_2.11-0.13.0.jar
-spire_2.11-0.13.0.jar
+spire-macros_2.12-0.13.0.jar
+spire_2.12-0.13.0.jar
stax-api-1.0-2.jar
stax2-api-3.1.4.jar
stream-2.9.6.jar
@@ -203,5 +203,5 @@ xbean-asm7-shaded-4.12.jar
xmlenc-0.52.jar
xz-1.5.jar
zjsonpatch-0.3.0.jar
-zookeeper-3.4.6.jar
+zookeeper-3.4.7.jar
zstd-jni-1.3.5-3.jar
diff --git a/dev/docker-images/Makefile b/dev/docker-images/Makefile
index ed3e3a5ee7687..168d386770655 100644
--- a/dev/docker-images/Makefile
+++ b/dev/docker-images/Makefile
@@ -17,9 +17,9 @@
.PHONY: all publish base python r
-BASE_IMAGE_NAME = palantirtechnologies/circle-spark-base:0.1.0
-PYTHON_IMAGE_NAME = palantirtechnologies/circle-spark-python:0.1.0
-R_IMAGE_NAME = palantirtechnologies/circle-spark-r:0.1.0
+BASE_IMAGE_NAME = palantirtechnologies/circle-spark-base:0.1.3
+PYTHON_IMAGE_NAME = palantirtechnologies/circle-spark-python:0.1.3
+R_IMAGE_NAME = palantirtechnologies/circle-spark-r:0.1.3
all: base python r
diff --git a/dev/docker-images/base/Dockerfile b/dev/docker-images/base/Dockerfile
index 0e84ec665fcd7..61293349a0db5 100644
--- a/dev/docker-images/base/Dockerfile
+++ b/dev/docker-images/base/Dockerfile
@@ -31,7 +31,7 @@ RUN mkdir -p /usr/share/man/man1 \
git \
locales sudo openssh-client ca-certificates tar gzip parallel \
net-tools netcat unzip zip bzip2 gnupg curl wget \
- openjdk-8-jdk rsync pandoc pandoc-citeproc flake8 \
+ openjdk-8-jdk rsync pandoc pandoc-citeproc flake8 tzdata \
&& rm -rf /var/lib/apt/lists/*
# If you update java, make sure this aligns
@@ -42,8 +42,9 @@ ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
RUN ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime
# Use unicode
-RUN locale-gen C.UTF-8 || true
-ENV LANG=C.UTF-8
+RUN locale-gen en_US.UTF-8
+ENV LANG=en_US.UTF-8
+ENV TZ=UTC
# install jq
RUN JQ_URL="https://circle-downloads.s3.amazonaws.com/circleci-images/cache/linux-amd64/jq-latest" \
diff --git a/dev/docker-images/python/Dockerfile b/dev/docker-images/python/Dockerfile
index cb44b373617da..bb039b7c5c8b6 100644
--- a/dev/docker-images/python/Dockerfile
+++ b/dev/docker-images/python/Dockerfile
@@ -34,12 +34,8 @@ RUN mkdir -p $(pyenv root)/versions \
RUN pyenv global our-miniconda/envs/python2 our-miniconda/envs/python3 \
&& pyenv rehash
-RUN $CONDA_BIN install -y -n python2 -c anaconda -c conda-forge xmlrunner \
- && $CONDA_BIN install -y -n python3 -c anaconda -c conda-forge xmlrunner \
- && $CONDA_BIN clean --all
-
# Expose pyenv globally
ENV PATH=$CIRCLE_HOME/.pyenv/shims:$PATH
-RUN PYENV_VERSION=our-miniconda/envs/python2 $CIRCLE_HOME/.pyenv/shims/pip install unishark \
- && PYENV_VERSION=our-miniconda/envs/python3 $CIRCLE_HOME/.pyenv/shims/pip install unishark
+RUN PYENV_VERSION=our-miniconda/envs/python2 $CIRCLE_HOME/.pyenv/shims/pip install unishark unittest-xml-reporting \
+ && PYENV_VERSION=our-miniconda/envs/python3 $CIRCLE_HOME/.pyenv/shims/pip install unishark unittest-xml-reporting
diff --git a/dev/lint-python b/dev/lint-python
index 27d87f6b56680..06816932e754a 100755
--- a/dev/lint-python
+++ b/dev/lint-python
@@ -1,5 +1,4 @@
#!/usr/bin/env bash
-
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -16,160 +15,242 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+# define test binaries + versions
+PYDOCSTYLE_BUILD="pydocstyle"
+MINIMUM_PYDOCSTYLE="3.0.0"
-SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
-SPARK_ROOT_DIR="$(dirname "$SCRIPT_DIR")"
-# Exclude auto-generated configuration file.
-PATHS_TO_CHECK="$( cd "$SPARK_ROOT_DIR" && find . -name "*.py" )"
-DOC_PATHS_TO_CHECK="$( cd "$SPARK_ROOT_DIR" && find . -name "*.py" | grep -vF 'functions.py' )"
-PYCODESTYLE_REPORT_PATH="$SPARK_ROOT_DIR/dev/pycodestyle-report.txt"
-PYDOCSTYLE_REPORT_PATH="$SPARK_ROOT_DIR/dev/pydocstyle-report.txt"
-PYLINT_REPORT_PATH="$SPARK_ROOT_DIR/dev/pylint-report.txt"
-PYLINT_INSTALL_INFO="$SPARK_ROOT_DIR/dev/pylint-info.txt"
-
-PYDOCSTYLEBUILD="pydocstyle"
-MINIMUM_PYDOCSTYLEVERSION="3.0.0"
-
-FLAKE8BUILD="flake8"
+FLAKE8_BUILD="flake8"
MINIMUM_FLAKE8="3.5.0"
-SPHINXBUILD=${SPHINXBUILD:=sphinx-build}
-SPHINX_REPORT_PATH="$SPARK_ROOT_DIR/dev/sphinx-report.txt"
+PYCODESTYLE_BUILD="pycodestyle"
+MINIMUM_PYCODESTYLE="2.4.0"
-cd "$SPARK_ROOT_DIR"
+SPHINX_BUILD="sphinx-build"
-# compileall: https://docs.python.org/2/library/compileall.html
-python -B -m compileall -q -l $PATHS_TO_CHECK > "$PYCODESTYLE_REPORT_PATH"
-compile_status="${PIPESTATUS[0]}"
+function compile_python_test {
+ local COMPILE_STATUS=
+ local COMPILE_REPORT=
+
+ if [[ ! "$1" ]]; then
+ echo "No python files found! Something is very wrong -- exiting."
+ exit 1;
+ fi
-# Get pycodestyle at runtime so that we don't rely on it being installed on the build server.
-# See: https://github.com/apache/spark/pull/1744#issuecomment-50982162
-# Updated to the latest official version of pep8. pep8 is formally renamed to pycodestyle.
-PYCODESTYLE_VERSION="2.4.0"
-PYCODESTYLE_SCRIPT_PATH="$SPARK_ROOT_DIR/dev/pycodestyle-$PYCODESTYLE_VERSION.py"
-PYCODESTYLE_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/PyCQA/pycodestyle/$PYCODESTYLE_VERSION/pycodestyle.py"
+ # compileall: https://docs.python.org/2/library/compileall.html
+ echo "starting python compilation test..."
+ COMPILE_REPORT=$( (python -B -mcompileall -q -l $1) 2>&1)
+ COMPILE_STATUS=$?
+
+ if [ $COMPILE_STATUS -ne 0 ]; then
+ echo "Python compilation failed with the following errors:"
+ echo "$COMPILE_REPORT"
+ echo "$COMPILE_STATUS"
+ exit "$COMPILE_STATUS"
+ else
+ echo "python compilation succeeded."
+ echo
+ fi
+}
-if [ ! -e "$PYCODESTYLE_SCRIPT_PATH" ]; then
- curl --silent -o "$PYCODESTYLE_SCRIPT_PATH" "$PYCODESTYLE_SCRIPT_REMOTE_PATH"
- curl_status="$?"
+function pycodestyle_test {
+ local PYCODESTYLE_STATUS=
+ local PYCODESTYLE_REPORT=
+ local RUN_LOCAL_PYCODESTYLE=
+ local VERSION=
+ local EXPECTED_PYCODESTYLE=
+ local PYCODESTYLE_SCRIPT_PATH="$SPARK_ROOT_DIR/dev/pycodestyle-$MINIMUM_PYCODESTYLE.py"
+ local PYCODESTYLE_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/PyCQA/pycodestyle/$MINIMUM_PYCODESTYLE/pycodestyle.py"
- if [ "$curl_status" -ne 0 ]; then
- echo "Failed to download pycodestyle.py from \"$PYCODESTYLE_SCRIPT_REMOTE_PATH\"."
- exit "$curl_status"
+ if [[ ! "$1" ]]; then
+ echo "No python files found! Something is very wrong -- exiting."
+ exit 1;
fi
-fi
-
-# Easy install pylint in /dev/pylint. To easy_install into a directory, the PYTHONPATH should
-# be set to the directory.
-# dev/pylint should be appended to the PATH variable as well.
-# Jenkins by default installs the pylint3 version, so for now this just checks the code quality
-# of python3.
-export "PYTHONPATH=$SPARK_ROOT_DIR/dev/pylint"
-export "PYLINT_HOME=$PYTHONPATH"
-export "PATH=$PYTHONPATH:$PATH"
-
-# There is no need to write this output to a file
-# first, but we do so so that the check status can
-# be output before the report, like with the
-# scalastyle and RAT checks.
-python "$PYCODESTYLE_SCRIPT_PATH" --config=dev/tox.ini $PATHS_TO_CHECK >> "$PYCODESTYLE_REPORT_PATH"
-pycodestyle_status="${PIPESTATUS[0]}"
-
-if [ "$compile_status" -eq 0 -a "$pycodestyle_status" -eq 0 ]; then
- lint_status=0
-else
- lint_status=1
-fi
-
-if [ "$lint_status" -ne 0 ]; then
- echo "pycodestyle checks failed."
- cat "$PYCODESTYLE_REPORT_PATH"
- rm "$PYCODESTYLE_REPORT_PATH"
- exit "$lint_status"
-else
- echo "pycodestyle checks passed."
- rm "$PYCODESTYLE_REPORT_PATH"
-fi
-
-# Check by flake8
-if hash "$FLAKE8BUILD" 2> /dev/null; then
- FLAKE8VERSION="$( $FLAKE8BUILD --version 2> /dev/null )"
- VERSION=($FLAKE8VERSION)
- IS_EXPECTED_FLAKE8=$(python -c 'from distutils.version import LooseVersion; \
-print(LooseVersion("""'${VERSION[0]}'""") >= LooseVersion("""'$MINIMUM_FLAKE8'"""))' 2> /dev/null)
- if [[ "$IS_EXPECTED_FLAKE8" == "True" ]]; then
- # stop the build if there are Python syntax errors or undefined names
- $FLAKE8BUILD . --count --select=E901,E999,F821,F822,F823 --max-line-length=100 --show-source --statistics
- flake8_status="${PIPESTATUS[0]}"
-
- if [ "$flake8_status" -eq 0 ]; then
- lint_status=0
- else
- lint_status=1
+
+ # check for locally installed pycodestyle & version
+ RUN_LOCAL_PYCODESTYLE="False"
+ if hash "$PYCODESTYLE_BUILD" 2> /dev/null; then
+ VERSION=$( $PYCODESTYLE_BUILD --version 2> /dev/null)
+ EXPECTED_PYCODESTYLE=$( (python -c 'from distutils.version import LooseVersion;
+ print(LooseVersion("""'${VERSION[0]}'""") >= LooseVersion("""'$MINIMUM_PYCODESTYLE'"""))')\
+ 2> /dev/null)
+
+ if [ "$EXPECTED_PYCODESTYLE" == "True" ]; then
+ RUN_LOCAL_PYCODESTYLE="True"
fi
+ fi
- if [ "$lint_status" -ne 0 ]; then
- echo "flake8 checks failed."
- exit "$lint_status"
- else
- echo "flake8 checks passed."
+ # download the right version or run locally
+ if [ $RUN_LOCAL_PYCODESTYLE == "False" ]; then
+ # Get pycodestyle at runtime so that we don't rely on it being installed on the build server.
+ # See: https://github.com/apache/spark/pull/1744#issuecomment-50982162
+ # Updated to the latest official version of pep8. pep8 is formally renamed to pycodestyle.
+ echo "downloading pycodestyle from $PYCODESTYLE_SCRIPT_REMOTE_PATH..."
+ if [ ! -e "$PYCODESTYLE_SCRIPT_PATH" ]; then
+ curl --silent -o "$PYCODESTYLE_SCRIPT_PATH" "$PYCODESTYLE_SCRIPT_REMOTE_PATH"
+ local curl_status="$?"
+
+ if [ "$curl_status" -ne 0 ]; then
+ echo "Failed to download pycodestyle.py from $PYCODESTYLE_SCRIPT_REMOTE_PATH"
+ exit "$curl_status"
+ fi
fi
+
+ echo "starting pycodestyle test..."
+ PYCODESTYLE_REPORT=$( (python "$PYCODESTYLE_SCRIPT_PATH" --config=dev/tox.ini $1) 2>&1)
+ PYCODESTYLE_STATUS=$?
+ else
+ # we have the right version installed, so run locally
+ echo "starting pycodestyle test..."
+ PYCODESTYLE_REPORT=$( ($PYCODESTYLE_BUILD --config=dev/tox.ini $1) 2>&1)
+ PYCODESTYLE_STATUS=$?
+ fi
+
+ if [ $PYCODESTYLE_STATUS -ne 0 ]; then
+ echo "pycodestyle checks failed:"
+ echo "$PYCODESTYLE_REPORT"
+ exit "$PYCODESTYLE_STATUS"
else
- echo "The flake8 version needs to be "$MINIMUM_FLAKE8" at latest. Your current version is '"$FLAKE8VERSION"'."
+ echo "pycodestyle checks passed."
+ echo
+ fi
+}
+
+function flake8_test {
+ local FLAKE8_VERSION=
+ local VERSION=
+ local EXPECTED_FLAKE8=
+ local FLAKE8_REPORT=
+ local FLAKE8_STATUS=
+
+ if ! hash "$FLAKE8_BUILD" 2> /dev/null; then
+ echo "The flake8 command was not found."
echo "flake8 checks failed."
exit 1
fi
-else
- echo >&2 "The flake8 command was not found."
- echo "flake8 checks failed."
- exit 1
-fi
-
-# Check python document style, skip check if pydocstyle is not installed.
-if hash "$PYDOCSTYLEBUILD" 2> /dev/null; then
- PYDOCSTYLEVERSION="$( $PYDOCSTYLEBUILD --version 2> /dev/null )"
- IS_EXPECTED_PYDOCSTYLEVERSION=$(python -c 'from distutils.version import LooseVersion; \
-print(LooseVersion("""'$PYDOCSTYLEVERSION'""") >= LooseVersion("""'$MINIMUM_PYDOCSTYLEVERSION'"""))')
- if [[ "$IS_EXPECTED_PYDOCSTYLEVERSION" == "True" ]]; then
- $PYDOCSTYLEBUILD --config=dev/tox.ini $DOC_PATHS_TO_CHECK >> "$PYDOCSTYLE_REPORT_PATH"
- pydocstyle_status="${PIPESTATUS[0]}"
-
- if [ "$compile_status" -eq 0 -a "$pydocstyle_status" -eq 0 ]; then
- echo "pydocstyle checks passed."
- rm "$PYDOCSTYLE_REPORT_PATH"
- else
- echo "pydocstyle checks failed."
- cat "$PYDOCSTYLE_REPORT_PATH"
- rm "$PYDOCSTYLE_REPORT_PATH"
- exit 1
- fi
+ FLAKE8_VERSION="$($FLAKE8_BUILD --version 2> /dev/null)"
+ VERSION=($FLAKE8_VERSION)
+ EXPECTED_FLAKE8=$( (python -c 'from distutils.version import LooseVersion;
+ print(LooseVersion("""'${VERSION[0]}'""") >= LooseVersion("""'$MINIMUM_FLAKE8'"""))') \
+ 2> /dev/null)
+
+ if [[ "$EXPECTED_FLAKE8" == "False" ]]; then
+ echo "\
+The minimum flake8 version needs to be $MINIMUM_FLAKE8. Your current version is $FLAKE8_VERSION
+
+flake8 checks failed."
+ exit 1
+ fi
+
+ echo "starting $FLAKE8_BUILD test..."
+ FLAKE8_REPORT=$( ($FLAKE8_BUILD . --count --select=E901,E999,F821,F822,F823 \
+ --max-line-length=100 --show-source --statistics) 2>&1)
+ FLAKE8_STATUS=$?
+
+ if [ "$FLAKE8_STATUS" -ne 0 ]; then
+ echo "flake8 checks failed:"
+ echo "$FLAKE8_REPORT"
+ echo "$FLAKE8_STATUS"
+ exit "$FLAKE8_STATUS"
else
- echo "The pydocstyle version needs to be "$MINIMUM_PYDOCSTYLEVERSION" at latest. Your current version is "$PYDOCSTYLEVERSION". Skipping pydoc checks for now."
+ echo "flake8 checks passed."
+ echo
fi
-else
- echo >&2 "The pydocstyle command was not found. Skipping pydoc checks for now"
-fi
-
-# Check that the documentation builds acceptably, skip check if sphinx is not installed.
-if hash "$SPHINXBUILD" 2> /dev/null; then
- cd python/docs
- make clean
- # Treat warnings as errors so we stop correctly
- SPHINXOPTS="-a -W" make html &> "$SPHINX_REPORT_PATH" || lint_status=1
- if [ "$lint_status" -ne 0 ]; then
- echo "pydoc checks failed."
- cat "$SPHINX_REPORT_PATH"
- echo "re-running make html to print full warning list"
- make clean
- SPHINXOPTS="-a" make html
- rm "$SPHINX_REPORT_PATH"
- exit "$lint_status"
- else
- echo "pydoc checks passed."
- rm "$SPHINX_REPORT_PATH"
- fi
- cd ../..
-else
- echo >&2 "The $SPHINXBUILD command was not found. Skipping pydoc checks for now"
-fi
+}
+
+function pydocstyle_test {
+ local PYDOCSTYLE_REPORT=
+ local PYDOCSTYLE_STATUS=
+ local PYDOCSTYLE_VERSION=
+ local EXPECTED_PYDOCSTYLE=
+
+ # Exclude auto-generated configuration file.
+ local DOC_PATHS_TO_CHECK="$( cd "${SPARK_ROOT_DIR}" && find . -name "*.py" | grep -vF 'functions.py' )"
+
+ # Check python document style, skip check if pydocstyle is not installed.
+ if ! hash "$PYDOCSTYLE_BUILD" 2> /dev/null; then
+ echo "The pydocstyle command was not found. Skipping pydocstyle checks for now."
+ echo
+ return
+ fi
+
+ PYDOCSTYLE_VERSION="$($PYDOCSTYLEBUILD --version 2> /dev/null)"
+ EXPECTED_PYDOCSTYLE=$(python -c 'from distutils.version import LooseVersion; \
+ print(LooseVersion("""'$PYDOCSTYLE_VERSION'""") >= LooseVersion("""'$MINIMUM_PYDOCSTYLE'"""))' \
+ 2> /dev/null)
+
+ if [[ "$EXPECTED_PYDOCSTYLE" == "False" ]]; then
+ echo "\
+The minimum version of pydocstyle needs to be $MINIMUM_PYDOCSTYLE.
+Your current version is $PYDOCSTYLE_VERSION.
+Skipping pydocstyle checks for now."
+ echo
+ return
+ fi
+
+ echo "starting $PYDOCSTYLE_BUILD test..."
+ PYDOCSTYLE_REPORT=$( ($PYDOCSTYLE_BUILD --config=dev/tox.ini $DOC_PATHS_TO_CHECK) 2>&1)
+ PYDOCSTYLE_STATUS=$?
+
+ if [ "$PYDOCSTYLE_STATUS" -ne 0 ]; then
+ echo "pydocstyle checks failed:"
+ echo "$PYDOCSTYLE_REPORT"
+ exit "$PYDOCSTYLE_STATUS"
+ else
+ echo "pydocstyle checks passed."
+ echo
+ fi
+}
+
+function sphinx_test {
+ local SPHINX_REPORT=
+ local SPHINX_STATUS=
+
+ # Check that the documentation builds acceptably, skip check if sphinx is not installed.
+ if ! hash "$SPHINX_BUILD" 2> /dev/null; then
+ echo "The $SPHINX_BUILD command was not found. Skipping pydoc checks for now."
+ echo
+ return
+ fi
+
+ echo "starting $SPHINX_BUILD tests..."
+ pushd python/docs &> /dev/null
+ make clean &> /dev/null
+ # Treat warnings as errors so we stop correctly
+ SPHINX_REPORT=$( (SPHINXOPTS="-a -W" make html) 2>&1)
+ SPHINX_STATUS=$?
+
+ if [ "$SPHINX_STATUS" -ne 0 ]; then
+ echo "$SPHINX_BUILD checks failed:"
+ echo "$SPHINX_REPORT"
+ echo
+ echo "re-running make html to print full warning list:"
+ make clean &> /dev/null
+ SPHINX_REPORT=$( (SPHINXOPTS="-a" make html) 2>&1)
+ echo "$SPHINX_REPORT"
+ exit "$SPHINX_STATUS"
+ else
+ echo "$SPHINX_BUILD checks passed."
+ echo
+ fi
+
+ popd &> /dev/null
+}
+
+SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
+SPARK_ROOT_DIR="$(dirname "${SCRIPT_DIR}")"
+
+pushd "$SPARK_ROOT_DIR" &> /dev/null
+
+PYTHON_SOURCE="$(find . -name "*.py")"
+
+compile_python_test "$PYTHON_SOURCE"
+pycodestyle_test "$PYTHON_SOURCE"
+flake8_test
+pydocstyle_test
+sphinx_test
+
+echo
+echo "all lint-python tests passed!"
+
+popd &> /dev/null
diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh
index e4405d89a1c17..1bde61b998ca5 100755
--- a/dev/make-distribution.sh
+++ b/dev/make-distribution.sh
@@ -173,7 +173,7 @@ fi
# Normal quoting tricks don't work.
# See: http://mywiki.wooledge.org/BashFAQ/050
if [[ -z "$DONT_BUILD" ]]; then
- BUILD_COMMAND=("$MVN" -T 1C $MAYBE_CLEAN package -DskipTests $@)
+ BUILD_COMMAND=("$MVN" $MAYBE_CLEAN package -DskipTests $@)
# Actually build the jar
echo -e "\nBuilding with..."
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 5db8efda90774..db44c3bff9083 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -310,6 +310,7 @@ def __hash__(self):
"python/(?!pyspark/(ml|mllib|sql|streaming))"
],
python_test_goals=[
+ # doctests
"pyspark.rdd",
"pyspark.context",
"pyspark.conf",
@@ -318,10 +319,22 @@ def __hash__(self):
"pyspark.serializers",
"pyspark.profiler",
"pyspark.shuffle",
- "pyspark.tests",
- "pyspark.test_broadcast",
- "pyspark.test_serializers",
"pyspark.util",
+ # unittests
+ "pyspark.tests.test_appsubmit",
+ "pyspark.tests.test_broadcast",
+ "pyspark.tests.test_conf",
+ "pyspark.tests.test_context",
+ "pyspark.tests.test_daemon",
+ "pyspark.tests.test_join",
+ "pyspark.tests.test_profiler",
+ "pyspark.tests.test_rdd",
+ "pyspark.tests.test_readwrite",
+ "pyspark.tests.test_serializers",
+ "pyspark.tests.test_shuffle",
+ "pyspark.tests.test_taskcontext",
+ "pyspark.tests.test_util",
+ "pyspark.tests.test_worker",
]
)
@@ -333,6 +346,7 @@ def __hash__(self):
"python/pyspark/sql"
],
python_test_goals=[
+ # doctests
"pyspark.sql.types",
"pyspark.sql.context",
"pyspark.sql.session",
@@ -346,7 +360,29 @@ def __hash__(self):
"pyspark.sql.streaming",
"pyspark.sql.udf",
"pyspark.sql.window",
- "pyspark.sql.tests",
+ # unittests
+ "pyspark.sql.tests.test_appsubmit",
+ "pyspark.sql.tests.test_arrow",
+ "pyspark.sql.tests.test_catalog",
+ "pyspark.sql.tests.test_column",
+ "pyspark.sql.tests.test_conf",
+ "pyspark.sql.tests.test_context",
+ "pyspark.sql.tests.test_dataframe",
+ "pyspark.sql.tests.test_datasources",
+ "pyspark.sql.tests.test_functions",
+ "pyspark.sql.tests.test_group",
+ "pyspark.sql.tests.test_pandas_udf",
+ "pyspark.sql.tests.test_pandas_udf_grouped_agg",
+ "pyspark.sql.tests.test_pandas_udf_grouped_map",
+ "pyspark.sql.tests.test_pandas_udf_scalar",
+ "pyspark.sql.tests.test_pandas_udf_window",
+ "pyspark.sql.tests.test_readwriter",
+ "pyspark.sql.tests.test_serde",
+ "pyspark.sql.tests.test_session",
+ "pyspark.sql.tests.test_streaming",
+ "pyspark.sql.tests.test_types",
+ "pyspark.sql.tests.test_udf",
+ "pyspark.sql.tests.test_utils",
]
)
@@ -362,8 +398,13 @@ def __hash__(self):
"python/pyspark/streaming"
],
python_test_goals=[
+ # doctests
"pyspark.streaming.util",
- "pyspark.streaming.tests",
+ # unittests
+ "pyspark.streaming.tests.test_context",
+ "pyspark.streaming.tests.test_dstream",
+ "pyspark.streaming.tests.test_kinesis",
+ "pyspark.streaming.tests.test_listener",
]
)
@@ -375,6 +416,7 @@ def __hash__(self):
"python/pyspark/mllib"
],
python_test_goals=[
+ # doctests
"pyspark.mllib.classification",
"pyspark.mllib.clustering",
"pyspark.mllib.evaluation",
@@ -389,7 +431,13 @@ def __hash__(self):
"pyspark.mllib.stat.KernelDensity",
"pyspark.mllib.tree",
"pyspark.mllib.util",
- "pyspark.mllib.tests",
+ # unittests
+ "pyspark.mllib.tests.test_algorithms",
+ "pyspark.mllib.tests.test_feature",
+ "pyspark.mllib.tests.test_linalg",
+ "pyspark.mllib.tests.test_stat",
+ "pyspark.mllib.tests.test_streaming_algorithms",
+ "pyspark.mllib.tests.test_util",
],
blacklisted_python_implementations=[
"PyPy" # Skip these tests under PyPy since they require numpy and it isn't available there
@@ -404,6 +452,7 @@ def __hash__(self):
"python/pyspark/ml/"
],
python_test_goals=[
+ # doctests
"pyspark.ml.classification",
"pyspark.ml.clustering",
"pyspark.ml.evaluation",
@@ -415,7 +464,20 @@ def __hash__(self):
"pyspark.ml.regression",
"pyspark.ml.stat",
"pyspark.ml.tuning",
- "pyspark.ml.tests",
+ # unittests
+ "pyspark.ml.tests.test_algorithms",
+ "pyspark.ml.tests.test_base",
+ "pyspark.ml.tests.test_evaluation",
+ "pyspark.ml.tests.test_feature",
+ "pyspark.ml.tests.test_image",
+ "pyspark.ml.tests.test_linalg",
+ "pyspark.ml.tests.test_param",
+ "pyspark.ml.tests.test_persistence",
+ "pyspark.ml.tests.test_pipeline",
+ "pyspark.ml.tests.test_stat",
+ "pyspark.ml.tests.test_training_summary",
+ "pyspark.ml.tests.test_tuning",
+ "pyspark.ml.tests.test_wrapper",
],
blacklisted_python_implementations=[
"PyPy" # Skip these tests under PyPy since they require numpy and it isn't available there
diff --git a/dev/test_functions.py b/dev/test_functions.py
index 3c9c7ccdefd25..14b038176e1c5 100755
--- a/dev/test_functions.py
+++ b/dev/test_functions.py
@@ -481,7 +481,7 @@ def parse_opts():
prog="run-tests"
)
parser.add_option(
- "-p", "--parallelism", type="int", default=4,
+ "-p", "--parallelism", type="int", default=8,
help="The number of suites to test in parallel (default %default)"
)
diff --git a/dists/hadoop-palantir-bom/pom.xml b/dists/hadoop-palantir-bom/pom.xml
index 4dcbb5149f750..523997699e9e6 100644
--- a/dists/hadoop-palantir-bom/pom.xml
+++ b/dists/hadoop-palantir-bom/pom.xml
@@ -20,12 +20,12 @@
4.0.0
org.apache.spark
- spark-parent_2.11
+ spark-parent_2.12
3.0.0-SNAPSHOT
../../pom.xml
- spark-dist_2.11-hadoop-palantir-bom
+ spark-dist_2.12-hadoop-palantir-bom
Spark Project Dist Palantir Hadoop (BOM)
http://spark.apache.org/
pom
@@ -107,7 +107,7 @@
org.apache.spark
- spark-avro_2.11
+ spark-avro_${scala.binary.version}
${project.version}
diff --git a/dists/hadoop-palantir/pom.xml b/dists/hadoop-palantir/pom.xml
index d3158b1bbef10..a3bb85e70f844 100644
--- a/dists/hadoop-palantir/pom.xml
+++ b/dists/hadoop-palantir/pom.xml
@@ -20,12 +20,12 @@
4.0.0
org.apache.spark
- spark-dist_2.11-hadoop-palantir-bom
+ spark-dist_2.12-hadoop-palantir-bom
3.0.0-SNAPSHOT
../hadoop-palantir-bom/pom.xml
- spark-dist_2.11-hadoop-palantir
+ spark-dist_2.12-hadoop-palantir
Spark Project Dist Palantir Hadoop
http://spark.apache.org/
pom
@@ -57,7 +57,7 @@
org.apache.spark
- spark-avro_2.11
+ spark-avro_${scala.binary.version}
org.apache.spark
diff --git a/docs/_config.yml b/docs/_config.yml
index c3ef98575fa62..649d18bf72b57 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -16,8 +16,8 @@ include:
# of Spark, Scala, and Mesos.
SPARK_VERSION: 3.0.0-SNAPSHOT
SPARK_VERSION_SHORT: 3.0.0
-SCALA_BINARY_VERSION: "2.11"
-SCALA_VERSION: "2.11.12"
+SCALA_BINARY_VERSION: "2.12"
+SCALA_VERSION: "2.12.7"
MESOS_VERSION: 1.0.0
SPARK_ISSUE_TRACKER_URL: https://issues.apache.org/jira/browse/SPARK
SPARK_GITHUB_URL: https://github.com/apache/spark
diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb
index 4d0d043a349bb..2d1a9547e3731 100644
--- a/docs/_plugins/copy_api_dirs.rb
+++ b/docs/_plugins/copy_api_dirs.rb
@@ -37,7 +37,7 @@
# Copy over the unified ScalaDoc for all projects to api/scala.
# This directory will be copied over to _site when `jekyll` command is run.
- source = "../target/scala-2.11/unidoc"
+ source = "../target/scala-2.12/unidoc"
dest = "api/scala"
puts "Making directory " + dest
diff --git a/docs/building-spark.md b/docs/building-spark.md
index 8af90db9a19dd..dfcd53c48e85c 100644
--- a/docs/building-spark.md
+++ b/docs/building-spark.md
@@ -96,9 +96,9 @@ It's possible to build Spark submodules using the `mvn -pl` option.
For instance, you can build the Spark Streaming module using:
- ./build/mvn -pl :spark-streaming_2.11 clean install
+ ./build/mvn -pl :spark-streaming_{{site.SCALA_BINARY_VERSION}} clean install
-where `spark-streaming_2.11` is the `artifactId` as defined in `streaming/pom.xml` file.
+where `spark-streaming_{{site.SCALA_BINARY_VERSION}}` is the `artifactId` as defined in `streaming/pom.xml` file.
## Continuous Compilation
@@ -230,7 +230,7 @@ Once installed, the `docker` service needs to be started, if not already running
On Linux, this can be done by `sudo service docker start`.
./build/mvn install -DskipTests
- ./build/mvn test -Pdocker-integration-tests -pl :spark-docker-integration-tests_2.11
+ ./build/mvn test -Pdocker-integration-tests -pl :spark-docker-integration-tests_{{site.SCALA_BINARY_VERSION}}
or
@@ -238,17 +238,17 @@ or
## Change Scala Version
-To build Spark using another supported Scala version, please change the major Scala version using (e.g. 2.12):
+To build Spark using another supported Scala version, please change the major Scala version using (e.g. 2.11):
- ./dev/change-scala-version.sh 2.12
+ ./dev/change-scala-version.sh 2.11
-For Maven, please enable the profile (e.g. 2.12):
+For Maven, please enable the profile (e.g. 2.11):
- ./build/mvn -Pscala-2.12 compile
+ ./build/mvn -Pscala-2.11 compile
-For SBT, specify a complete scala version using (e.g. 2.12.6):
+For SBT, specify a complete scala version using (e.g. 2.11.12):
- ./build/sbt -Dscala.version=2.12.6
+ ./build/sbt -Dscala.version=2.11.12
Otherwise, the sbt-pom-reader plugin will use the `scala.version` specified in the spark-parent pom.
diff --git a/docs/cloud-integration.md b/docs/cloud-integration.md
index fc24fa3ec4871..ccbbaaf886886 100644
--- a/docs/cloud-integration.md
+++ b/docs/cloud-integration.md
@@ -85,7 +85,7 @@ is set to the chosen version of Spark:
...
org.apache.spark
- spark-hadoop-cloud_2.11
+ hadoop-cloud_{{site.SCALA_BINARY_VERSION}}
${spark.version}
...
diff --git a/docs/configuration.md b/docs/configuration.md
index f8937b0bc61a0..04210d855b110 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -266,6 +266,39 @@ of the most common options to set are:
Only has effect in Spark standalone mode or Mesos cluster deploy mode.
|
+