From cf8b6cf5ac91673959e9d1200975cd23cc5f4613 Mon Sep 17 00:00:00 2001 From: Ashwin Shankar Date: Thu, 7 May 2015 16:38:30 -0700 Subject: [PATCH 1/2] Stop counting preemption of executors as failure --- .../scala/org/apache/spark/deploy/yarn/YarnAllocator.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala index b8f42dadcb464..acdced5d44823 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala @@ -373,7 +373,11 @@ private[yarn] class YarnAllocator( // Hadoop 2.2.X added a ContainerExitStatus we should switch to use // there are some exit status' we shouldn't necessarily count against us, but for // now I think its ok as none of the containers are expected to exit - if (completedContainer.getExitStatus == -103) { // vmem limit exceeded + if (completedContainer.getExitStatus == ContainerExitStatus.PREEMPTED) { + logInfo("Container killed: " + containerId + + ". Exit status: " + completedContainer.getExitStatus + + ". Diagnostics: " + completedContainer.getDiagnostics) + } else if (completedContainer.getExitStatus == -103) { // vmem limit exceeded logWarning(memLimitExceededLogMessage( completedContainer.getDiagnostics, VMEM_EXCEEDED_PATTERN)) From 90900cf23da0a82394f534d4c28fa30f1e860630 Mon Sep 17 00:00:00 2001 From: Ashwin Shankar Date: Fri, 8 May 2015 10:03:32 -0700 Subject: [PATCH 2/2] Fix log info message --- .../scala/org/apache/spark/deploy/yarn/YarnAllocator.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala index acdced5d44823..88d68d5556162 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala @@ -374,9 +374,7 @@ private[yarn] class YarnAllocator( // there are some exit status' we shouldn't necessarily count against us, but for // now I think its ok as none of the containers are expected to exit if (completedContainer.getExitStatus == ContainerExitStatus.PREEMPTED) { - logInfo("Container killed: " + containerId + - ". Exit status: " + completedContainer.getExitStatus + - ". Diagnostics: " + completedContainer.getDiagnostics) + logInfo("Container preempted: " + containerId) } else if (completedContainer.getExitStatus == -103) { // vmem limit exceeded logWarning(memLimitExceededLogMessage( completedContainer.getDiagnostics,