From fa40db011e9dd9e67482d5a659103196d5a6c8a6 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Fri, 19 Dec 2014 10:27:42 +0800 Subject: [PATCH] Add an Ordering for NullWritable to make the compiler generate same byte codes for RDD --- core/src/main/scala/org/apache/spark/rdd/RDD.scala | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala index 214f22bc5b603..4bdb13f9ad3d0 100644 --- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala @@ -1174,6 +1174,14 @@ abstract class RDD[T: ClassTag]( * Save this RDD as a text file, using string representations of elements. */ def saveAsTextFile(path: String) { + // https://issues.apache.org/jira/browse/SPARK-2075 + // NullWritable is a Comparable rather than Comparable[NullWritable] in Hadoop 1.+, + // so the compiler cannot find an implicit Ordering for it. It will generate different + // anonymous classes for `saveAsTextFile` in Hadoop 1.+ and Hadoop 2.+. Therefore, here we + // provide an Ordering for NullWritable so that the compiler will generate same codes. + implicit val nullWritableOrdering = new Ordering[NullWritable] { + override def compare(x: NullWritable, y: NullWritable): Int = 0 + } this.map(x => (NullWritable.get(), new Text(x.toString))) .saveAsHadoopFile[TextOutputFormat[NullWritable, Text]](path) } @@ -1182,6 +1190,10 @@ abstract class RDD[T: ClassTag]( * Save this RDD as a compressed text file, using string representations of elements. */ def saveAsTextFile(path: String, codec: Class[_ <: CompressionCodec]) { + // https://issues.apache.org/jira/browse/SPARK-2075 + implicit val nullWritableOrdering = new Ordering[NullWritable] { + override def compare(x: NullWritable, y: NullWritable): Int = 0 + } this.map(x => (NullWritable.get(), new Text(x.toString))) .saveAsHadoopFile[TextOutputFormat[NullWritable, Text]](path, codec) }