Skip to content

Commit

Permalink
Add an Ordering for NullWritable to make the compiler generate same b…
Browse files Browse the repository at this point in the history
…yte codes for RDD
  • Loading branch information
zsxwing committed Dec 19, 2014
1 parent 9804a75 commit fa40db0
Showing 1 changed file with 12 additions and 0 deletions.
12 changes: 12 additions & 0 deletions core/src/main/scala/org/apache/spark/rdd/RDD.scala
Original file line number Diff line number Diff line change
Expand Up @@ -1174,6 +1174,14 @@ abstract class RDD[T: ClassTag](
* Save this RDD as a text file, using string representations of elements.
*/
def saveAsTextFile(path: String) {
// https://issues.apache.org/jira/browse/SPARK-2075
// NullWritable is a Comparable rather than Comparable[NullWritable] in Hadoop 1.+,
// so the compiler cannot find an implicit Ordering for it. It will generate different
// anonymous classes for `saveAsTextFile` in Hadoop 1.+ and Hadoop 2.+. Therefore, here we
// provide an Ordering for NullWritable so that the compiler will generate same codes.
implicit val nullWritableOrdering = new Ordering[NullWritable] {
override def compare(x: NullWritable, y: NullWritable): Int = 0
}
this.map(x => (NullWritable.get(), new Text(x.toString)))
.saveAsHadoopFile[TextOutputFormat[NullWritable, Text]](path)
}
Expand All @@ -1182,6 +1190,10 @@ abstract class RDD[T: ClassTag](
* Save this RDD as a compressed text file, using string representations of elements.
*/
def saveAsTextFile(path: String, codec: Class[_ <: CompressionCodec]) {
// https://issues.apache.org/jira/browse/SPARK-2075
implicit val nullWritableOrdering = new Ordering[NullWritable] {
override def compare(x: NullWritable, y: NullWritable): Int = 0
}
this.map(x => (NullWritable.get(), new Text(x.toString)))
.saveAsHadoopFile[TextOutputFormat[NullWritable, Text]](path, codec)
}
Expand Down

0 comments on commit fa40db0

Please sign in to comment.