added todo explaining cost of creating Row object in python

pdeyhim · Apr 15, 2014 · 906d180 · 906d180
1 parent 251f99d
commit 906d180
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py
@@ -1419,6 +1419,9 @@ def registerAsTable(self, name):
 
     def toPython(self):
         jrdd = self._jschema_rdd.javaToPython()
+        # TODO: This is inefficient, we should construct the Python Row object
+        # in Java land in the javaToPython function. May require a custom
+        # pickle serializer in Pyrolite
         return RDD(jrdd, self._sc, self._sc.serializer).map(lambda d: Row(d))
 
 def _test():