[SPARK-2014] Make PySpark store RDDs in MEMORY_ONLY_SER with compress…

…ion by default Author: Prashant Sharma <[email protected]> Closes #1051 from ScrapCodes/SPARK-2014/pyspark-cache and squashes the following commits: f192df7 [Prashant Sharma] Code Review 2a2f43f [Prashant Sharma] [SPARK-2014] Make PySpark store RDDs in MEMORY_ONLY_SER with compression by default
apache · Jul 25, 2014 · eff9714 · eff9714
1 parent a45d548
commit eff9714
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 3 deletions.
diff --git a/python/pyspark/conf.py b/python/pyspark/conf.py
@@ -100,6 +100,12 @@ def set(self, key, value):
         self._jconf.set(key, unicode(value))
         return self
 
+    def setIfMissing(self, key, value):
+        """Set a configuration property, if not already set."""
+        if self.get(key) is None:
+            self.set(key, value)
+        return self
+
     def setMaster(self, value):
         """Set master URL to connect to."""
         self._jconf.setMaster(value)

diff --git a/python/pyspark/context.py b/python/pyspark/context.py
@@ -101,7 +101,7 @@ def __init__(self, master=None, appName=None, sparkHome=None, pyFiles=None,
         else:
             self.serializer = BatchedSerializer(self._unbatched_serializer,
                                                 batchSize)
-
+        self._conf.setIfMissing("spark.rdd.compress", "true")
         # Set any parameters passed directly to us on the conf
         if master:
             self._conf.setMaster(master)

diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py
@@ -231,10 +231,10 @@ def context(self):
 
     def cache(self):
         """
-        Persist this RDD with the default storage level (C{MEMORY_ONLY}).
+        Persist this RDD with the default storage level (C{MEMORY_ONLY_SER}).
         """
         self.is_cached = True
-        self._jrdd.cache()
+        self.persist(StorageLevel.MEMORY_ONLY_SER)
         return self
 
     def persist(self, storageLevel):