Py3 compat: attempt to fix issues that caused failing unit tests

.travis.yml

@@ -5,7 +5,7 @@ language: python
 
 python:
   - "2.7"
-  #- "3.5"
+  - "3.5"
 
 # This tests against all the versions of DB supported by Eden
 env:

VERSION

@@ -1 +1 @@
-nursix-dev-2967-g0f2326b (2019-07-09 13:40:51)
+nursix-dev-2968-gcd9f908 (2019-07-09 15:07:54)

modules/s3/s3import.py

@@ -63,7 +63,7 @@
 from .s3rest import S3Method, S3Request
 from .s3resource import S3Resource
 from .s3utils import s3_auth_user_represent_name, s3_get_foreign_key, \
-                     s3_has_foreign_key, s3_mark_required, s3_unicode
+                     s3_has_foreign_key, s3_mark_required, s3_str, s3_unicode
 from .s3validators import IS_JSONS3
 
 # =============================================================================
@@ -2127,7 +2127,7 @@ def validate(self):
                     form.errors[k] = "[%s] %s" % (k, form.errors[k])
                 else:
                     e = e[0]
-                e.set(ERROR, str(form.errors[k]).decode("utf-8"))
+                e.set(ERROR, s3_unicode(form.errors[k]))
             self.error = current.ERROR.VALIDATION_ERROR
             accepted = False
 
@@ -3849,7 +3849,7 @@ def match(self, field, value):
             #    http://stackoverflow.com/questions/18507589/the-lower-function-on-international-characters-in-postgresql
             # => works fine on Debian servers if the locale is a .UTF-8 before
             #    the Postgres cluster is created
-            query = (field.lower() == s3_unicode(value).lower().encode("utf-8"))
+            query = (field.lower() == s3_str(s3_unicode(value).lower()))
         else:
             query = (field == value)
 

modules/s3/s3sync.py

@@ -36,7 +36,7 @@
 from gluon import current, URL, DIV
 from gluon.storage import Storage
 
-from s3compat import StringIO, unicodeT
+from s3compat import BytesIO, unicodeT
 from .s3datetime import s3_parse_datetime, s3_utc
 from .s3rest import S3Method
 from .s3import import S3ImportItem
@@ -1196,15 +1196,15 @@ def __init__(self, fileobj=None, compressed=True):
         if fileobj is not None:
             if not hasattr(fileobj, "seek"):
                 # Possibly a addinfourl instance from urlopen,
-                # => must copy to StringIO buffer for random access
-                fileobj = StringIO(fileobj.read())
+                # => must copy to BytesIO buffer for random access
+                fileobj = BytesIO(fileobj.read())
             try:
                 archive = zipfile.ZipFile(fileobj, "r")
             except RuntimeError:
                 current.log.warn("invalid ZIP archive: %s" % sys.exc_info()[1])
                 archive = None
         else:
-            fileobj = StringIO()
+            fileobj = BytesIO()
             try:
                 archive = zipfile.ZipFile(fileobj, "w", compression, True)
             except RuntimeError:
@@ -1248,12 +1248,12 @@ def add(self, name, obj):
                 obj.seek(0)
             archive.writestr(name, obj.read())
 
-        elif type(obj) is str:
+        elif isinstance(obj, (str, bytes)):
             archive.writestr(name, obj)
 
         elif isinstance(obj, unicodeT):
             # Convert unicode objects to str (Py2 backwards-compatibility)
-            archive.writestr(name, obj.encode("utf-8"))
+            archive.writestr(name, s3_str(obj))
 
         else:
             raise TypeError("invalid object type")

modules/s3/s3xml.py

@@ -341,13 +341,17 @@ def tostring(tree, xml_declaration=True, pretty_print=True):
             @param tree: the element tree
             @param xml_declaration: add an XML declaration to the output
             @param pretty_print: provide pretty formatted output
+
+            @returns: the XML as str
         """
 
-        return etree.tostring(tree,
-                              xml_declaration = xml_declaration,
-                              encoding = "utf-8" if PY2 else "unicode",
-                              pretty_print = pretty_print,
-                              )
+        string = etree.tostring(tree,
+                                xml_declaration = xml_declaration,
+                                encoding = "utf-8",
+                                pretty_print = pretty_print,
+                                )
+
+        return string
 
     # -------------------------------------------------------------------------
     def tree(self, elements,
@@ -1548,7 +1552,7 @@ def record(cls,
                     # Download file from network location
                     if not isinstance(download_url, str):
                         try:
-                            download_url = download_url.encode("utf-8")
+                            download_url = s3_str(download_url)
                         except UnicodeEncodeError:
                             continue
                     if not filename:
@@ -1561,7 +1565,7 @@ def record(cls,
                 if upload:
                     if not isinstance(filename, str):
                         try:
-                            filename = filename.encode("utf-8")
+                            filename = s3_str(filename)
                         except UnicodeEncodeError:
                             continue
                     field = table[f]
@@ -1616,7 +1620,7 @@ def record(cls,
                     if not isinstance(value, (basestring, list, tuple, bool)):
                         v = str(value)
                     elif isinstance(value, basestring):
-                        v = value.encode("utf-8")
+                        v = s3_str(value)
                     else:
                         v = value
                     filename = None

modules/s3compat.py

@@ -39,6 +39,7 @@
         from cStringIO import StringIO  # faster, where available
     except ImportError:
         from StringIO import StringIO
+    BytesIO = StringIO
     import urlparse
     import urllib2
     from urllib2 import HTTPError, URLError, urlopen
@@ -62,7 +63,7 @@
     from itertools import izip_longest as zip_longest
 else:
     import pickle
-    from io import StringIO
+    from io import StringIO, BytesIO
     from urllib import parse as urlparse
     from urllib import request as urllib2
     from urllib.error import HTTPError, URLError

modules/unit_tests/modules/s3layouts_tests.py

@@ -67,7 +67,7 @@ def testPopupLink(self):
         auth.s3_impersonate("[email protected]")
         self.assertTrue(comment.check_permission())
         output = comment.xml()
-        self.assertTrue(type(output) is str)
+        self.assertTrue(type(output) is bytes)
         self.assertNotEqual(output, "")
         auth.s3_impersonate(None)
 

modules/unit_tests/s3/s3resource.py

@@ -14,8 +14,8 @@
 from gluon import *
 from gluon.storage import Storage
 
-from s3dal import Row
 from s3 import *
+from s3dal import Row
 
 from unit_tests import run_suite
 

modules/unit_tests/s3/s3sync.py

@@ -12,7 +12,9 @@
 from lxml import etree
 
 from unit_tests import run_suite
+
 from s3 import S3SyncDataArchive
+from s3compat import PY2
 
 # =============================================================================
 class ExportMergeTests(unittest.TestCase):
@@ -444,8 +446,8 @@ def testArchive(self):
         archive = S3SyncDataArchive()
 
         # Add two XML strings to it
-        xmlstr1 = "<example>First Example</example>"
-        xmlstr2 = "<example>Second Example</example>"
+        xmlstr1 = b"<example>First Example</example>"
+        xmlstr2 = b"<example>Second Example</example>"
         archive.add("test1.xml", xmlstr1)
         archive.add("test2.xml", xmlstr2)
 

modules/unit_tests/s3/s3xml.py

@@ -14,7 +14,7 @@
 from gluon import *
 
 from s3 import S3Hierarchy, s3_meta_fields, S3Represent, S3RepresentLazy, S3XMLFormat, IS_ONE_OF
-from s3compat import StringIO
+from s3compat import BytesIO, StringIO
 
 from unit_tests import run_suite
 
@@ -1066,7 +1066,7 @@ def testForbiddenFileAccess(self):
 
         xml = current.xml
 
-        tree = xml.parse(StringIO(self.forbidden))
+        tree = xml.parse(BytesIO(self.forbidden.encode("utf-8")))
         self.assertEqual(tree, None)
         self.assertNotEqual(xml.error, None)
 
@@ -1076,7 +1076,7 @@ def testPermissibleAbsolutePathAccess(self):
 
         xml = current.xml
 
-        tree = xml.parse(StringIO(self.abspath))
+        tree = xml.parse(BytesIO(self.abspath.encode("utf-8")))
         self.assertNotEqual(tree, None)
         self.assertEqual(xml.error, None)
 
@@ -1086,7 +1086,7 @@ def testPermissibleRelativePathAccess(self):
 
         xml = current.xml
 
-        tree = xml.parse(StringIO(self.relpath))
+        tree = xml.parse(BytesIO(self.relpath.encode("utf-8")))
         self.assertNotEqual(tree, None)
         self.assertEqual(xml.error, None)
 
@@ -1097,7 +1097,7 @@ def testImportWithForbiddenFileAccess(self):
         resource = current.s3db.resource("org_organisation")
 
         with self.assertRaises(SyntaxError):
-            resource.import_xml(StringIO(self.forbidden))
+            resource.import_xml(BytesIO(self.forbidden.encode("utf-8")))
 
 # =============================================================================
 if __name__ == "__main__":