diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala index 40c8afb106824..f53823f2765dc 100644 --- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala +++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala @@ -1235,13 +1235,13 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)]) writer.commit() } - + lazy val result = writer.commitJob() self.context.submitJobWithTaskContext( self, writeToFile, 0 until self.partitions.size, (_, _: Unit) => {}, - { writer.commitJob() } + { result } ) } @@ -1330,12 +1330,13 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)]) val jobCommitter = jobFormat.getOutputCommitter(jobTaskContext) jobCommitter.setupJob(jobTaskContext) + lazy val result = jobCommitter.commitJob(jobTaskContext) self.context.submitJobWithTaskContext( self, writeShard, 0 until self.partitions.size, (_, _:Int) => {}, - { jobCommitter.commitJob(jobTaskContext) } + { result } ) }