SPARK-6045 RecordWriter should be checked against null in PairRDDFunc…

…tions#saveAsNewAPIHadoopDataset
apache · Feb 26, 2015 · 2632a57 · 2632a57
1 parent 2d8d4b1
commit 2632a57
Showing 1 changed file with 2 additions and 3 deletions.
diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
@@ -993,6 +993,7 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)])
       val (outputMetrics, bytesWrittenCallback) = initHadoopOutputMetrics(context)
 
       val writer = format.getRecordWriter(hadoopContext).asInstanceOf[NewRecordWriter[K,V]]
+      require(writer != null, s"Unable to obtain NewRecordWriter")
       var recordsWritten = 0L
       try {
         while (iter.hasNext) {
@@ -1004,9 +1005,7 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)])
           recordsWritten += 1
         }
       } finally {
-        if (writer != null) {
-          writer.close(hadoopContext)
-        }
+        writer.close(hadoopContext)
       }
       committer.commitTask(hadoopContext)
       bytesWrittenCallback.foreach { fn => outputMetrics.setBytesWritten(fn()) }