From e3dfd221dbdd1bd3ba0226cc7b9cafe939cd1676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Bry=C5=84ski?= Date: Fri, 15 Sep 2017 15:02:49 +0200 Subject: [PATCH 1/3] Change function call to references --- python/pyspark/sql/types.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py index 920cf009f599d..f37926a916ce4 100644 --- a/python/pyspark/sql/types.py +++ b/python/pyspark/sql/types.py @@ -410,6 +410,9 @@ def __init__(self, name, dataType, nullable=True, metadata=None): self.dataType = dataType self.nullable = nullable self.metadata = metadata or {} + self.needConversion = dataType.needConversion + self.toInternal = dataType.toInternal + self.fromInternal = dataType.fromInternal def simpleString(self): return '%s:%s' % (self.name, self.dataType.simpleString()) @@ -431,15 +434,6 @@ def fromJson(cls, json): json["nullable"], json["metadata"]) - def needConversion(self): - return self.dataType.needConversion() - - def toInternal(self, obj): - return self.dataType.toInternal(obj) - - def fromInternal(self, obj): - return self.dataType.fromInternal(obj) - def typeName(self): raise TypeError( "StructField does not have typeName. " From 2e6160efc41ea62b54c2ba34af5afa7729f30a3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Bry=C5=84ski?= Date: Fri, 15 Sep 2017 15:39:13 +0200 Subject: [PATCH 2/3] Custom pickling for StructField --- python/pyspark/sql/types.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py index f37926a916ce4..0727dd8573450 100644 --- a/python/pyspark/sql/types.py +++ b/python/pyspark/sql/types.py @@ -413,6 +413,21 @@ def __init__(self, name, dataType, nullable=True, metadata=None): self.needConversion = dataType.needConversion self.toInternal = dataType.toInternal self.fromInternal = dataType.fromInternal + + def __getstate__(self): + """Return state values to be pickled.""" + return (self.name, self.dataType, self.nullable, self.metadata) + + def __setstate__(self, state): + """Restore state from the unpickled state values.""" + name, dataType, nullable, metadata = state + self.name = name + self.dataType = dataType + self.nullable = nullable + self.metadata = metadata + self.needConversion = dataType.needConversion + self.toInternal = dataType.toInternal + self.fromInternal = dataType.fromInternal def simpleString(self): return '%s:%s' % (self.name, self.dataType.simpleString()) From f8e0ac426e6bf810c6485016b32a673147082e09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Bry=C5=84ski?= Date: Fri, 15 Sep 2017 15:45:08 +0200 Subject: [PATCH 3/3] PEP8 fix --- python/pyspark/sql/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py index 0727dd8573450..3f5907beb5902 100644 --- a/python/pyspark/sql/types.py +++ b/python/pyspark/sql/types.py @@ -413,7 +413,7 @@ def __init__(self, name, dataType, nullable=True, metadata=None): self.needConversion = dataType.needConversion self.toInternal = dataType.toInternal self.fromInternal = dataType.fromInternal - + def __getstate__(self): """Return state values to be pickled.""" return (self.name, self.dataType, self.nullable, self.metadata)