[WIP][CORE UPDATE - PART X] Protobuf_cpp fixes and updated version (#…

…1551)

* Update protobuf_cpp recipe and grant python3 compatibility

The removed flags are already set in base class, so no need to set in here.

* Move libraries from LDFLAGS to LIBS for protobuf_cpp recipe

Because this is how you are supposed to do it, you must use LDFLAGS for linker flags and LDLIBS (or the equivalent LOADLIBES) for the libraries

pythonforandroid/recipes/protobuf_cpp/__init__.py

@@ -1,7 +1,7 @@
 from pythonforandroid.recipe import PythonRecipe
 from pythonforandroid.logger import shprint, info_notify
 from pythonforandroid.util import current_directory, shutil
-from os.path import exists, join, dirname
+from os.path import exists, join
 import sh
 from multiprocessing import cpu_count
 from pythonforandroid.toolchain import info
@@ -11,7 +11,7 @@
 
 class ProtobufCppRecipe(PythonRecipe):
     name = 'protobuf_cpp'
-    version = '3.5.1'
+    version = '3.6.1'
     url = 'https://github.com/google/protobuf/releases/download/v{version}/protobuf-python-{version}.tar.gz'
     call_hostpython_via_targetpython = False
     depends = ['cffi', 'setuptools']
@@ -20,6 +20,12 @@ class ProtobufCppRecipe(PythonRecipe):
 
     def prebuild_arch(self, arch):
         super(ProtobufCppRecipe, self).prebuild_arch(arch)
+
+        patch_mark = join(self.get_build_dir(arch.arch), '.protobuf-patched')
+        if self.ctx.python_recipe.name == 'python3' and not exists(patch_mark):
+            self.apply_patch('fix-python3-compatibility.patch', arch.arch)
+            shprint(sh.touch, patch_mark)
+
         # During building, host needs to transpile .proto files to .py
         # ideally with the same version as protobuf runtime, or with an older one.
         # Because protoc is compiled for target (i.e. Android), we need an other binary
@@ -100,34 +106,18 @@ def install_python_package(self, arch):
         with current_directory(join(self.get_build_dir(arch.arch), 'python')):
             hostpython = sh.Command(self.hostpython_location)
 
-            if self.ctx.python_recipe.from_crystax:
-                hpenv = env.copy()
-                shprint(hostpython, 'setup.py', 'install', '-O2',
-                        '--root={}'.format(self.ctx.get_python_install_dir()),
-                        '--install-lib=.',
-                        '--cpp_implementation',
-                        _env=hpenv, *self.setup_extra_args)
-            else:
-                hppath = join(dirname(self.hostpython_location), 'Lib',
-                              'site-packages')
-                hpenv = env.copy()
-                if 'PYTHONPATH' in hpenv:
-                    hpenv['PYTHONPATH'] = ':'.join([hppath] +
-                                                   hpenv['PYTHONPATH'].split(':'))
-                else:
-                    hpenv['PYTHONPATH'] = hppath
-                shprint(hostpython, 'setup.py', 'install', '-O2',
-                        '--root={}'.format(self.ctx.get_python_install_dir()),
-                        '--install-lib=lib/python2.7/site-packages',
-                        '--cpp_implementation',
-                        _env=hpenv, *self.setup_extra_args)
+            hpenv = env.copy()
+            shprint(hostpython, 'setup.py', 'install', '-O2',
+                    '--root={}'.format(self.ctx.get_python_install_dir()),
+                    '--install-lib=.',
+                    '--cpp_implementation',
+                    _env=hpenv, *self.setup_extra_args)
 
     def get_recipe_env(self, arch):
         env = super(ProtobufCppRecipe, self).get_recipe_env(arch)
         if self.protoc_dir is not None:
             # we need protoc with binary for host platform
             env['PROTOC'] = join(self.protoc_dir, 'bin', 'protoc')
-        env['PYTHON_ROOT'] = self.ctx.get_python_install_dir()
         env['TARGET_OS'] = 'OS_ANDROID_CROSSCOMPILE'
         env['CFLAGS'] += (
             ' -I' + self.ctx.ndk_dir + '/platforms/android-' +
@@ -136,17 +126,17 @@ def get_recipe_env(self, arch):
             ' -I' + self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' +
             self.ctx.toolchain_version + '/include' +
             ' -I' + self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' +
-            self.ctx.toolchain_version + '/libs/' + arch.arch + '/include' +
-            ' -I' + env['PYTHON_ROOT'] + '/include/python2.7')
+            self.ctx.toolchain_version + '/libs/' + arch.arch + '/include')
+        env['CFLAGS'] += ' -std=gnu++11'
         env['CXXFLAGS'] = env['CFLAGS']
         env['CXXFLAGS'] += ' -frtti'
         env['CXXFLAGS'] += ' -fexceptions'
         env['LDFLAGS'] += (
             ' -L' + self.ctx.ndk_dir +
             '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version +
-            '/libs/' + arch.arch + ' -lgnustl_shared -lpython2.7 -landroid -llog')
+            '/libs/' + arch.arch)
+        env['LIBS'] = env.get('LIBS', '') + ' -lgnustl_shared -landroid -llog'
 
-        env['LDSHARED'] = env['CC'] + ' -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions'
         return env
 
 

pythonforandroid/recipes/protobuf_cpp/fix-python3-compatibility.patch

@@ -0,0 +1,91 @@
+From 539bc017a62f91bdf7c547b58948cb5a2f59d918 Mon Sep 17 00:00:00 2001
+From: Ben Webb <[email protected]>
+Date: Thu, 12 Jul 2018 10:58:10 -0700
+Subject: [PATCH] Add Python 3.7 compatibility (#4862)
+
+Compilation of Python wrappers fails with Python 3.7 because
+the Python folks changed their C API such that
+PyUnicode_AsUTF8AndSize() now returns a const char* rather
+than a char*. Add a patch to work around. Relates #4086.
+---
+ python/google/protobuf/pyext/descriptor.cc            | 2 +-
+ python/google/protobuf/pyext/descriptor_containers.cc | 2 +-
+ python/google/protobuf/pyext/descriptor_pool.cc       | 2 +-
+ python/google/protobuf/pyext/extension_dict.cc        | 2 +-
+ python/google/protobuf/pyext/message.cc               | 4 ++--
+ 5 files changed, 6 insertions(+), 6 deletions(-)
+
+diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc
+index 8af0cb1289..19a1c38a62 100644
+--- a/python/google/protobuf/pyext/descriptor.cc
++++ b/python/google/protobuf/pyext/descriptor.cc
+@@ -56,7 +56,7 @@
+   #endif
+   #define PyString_AsStringAndSize(ob, charpp, sizep) \
+     (PyUnicode_Check(ob)? \
+-       ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \
++       ((*(charpp) = const_cast<char*>(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \
+        PyBytes_AsStringAndSize(ob, (charpp), (sizep)))
+ #endif
+
+diff --git a/python/google/protobuf/pyext/descriptor_containers.cc b/python/google/protobuf/pyext/descriptor_containers.cc
+index bc007f7efa..0153664f50 100644
+--- a/python/google/protobuf/pyext/descriptor_containers.cc
++++ b/python/google/protobuf/pyext/descriptor_containers.cc
+@@ -66,7 +66,7 @@
+   #endif
+   #define PyString_AsStringAndSize(ob, charpp, sizep) \
+     (PyUnicode_Check(ob)? \
+-       ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \
++       ((*(charpp) = const_cast<char*>(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \
+        PyBytes_AsStringAndSize(ob, (charpp), (sizep)))
+ #endif
+
+diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc
+index 95882aeb35..962accc6e9 100644
+--- a/python/google/protobuf/pyext/descriptor_pool.cc
++++ b/python/google/protobuf/pyext/descriptor_pool.cc
+@@ -48,7 +48,7 @@
+   #endif
+   #define PyString_AsStringAndSize(ob, charpp, sizep) \
+     (PyUnicode_Check(ob)? \
+-       ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \
++       ((*(charpp) = const_cast<char*>(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \
+        PyBytes_AsStringAndSize(ob, (charpp), (sizep)))
+ #endif
+
+diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc
+index 018b5c2c49..174c5470c2 100644
+--- a/python/google/protobuf/pyext/extension_dict.cc
++++ b/python/google/protobuf/pyext/extension_dict.cc
+@@ -53,7 +53,7 @@
+   #endif
+   #define PyString_AsStringAndSize(ob, charpp, sizep) \
+     (PyUnicode_Check(ob)? \
+-       ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \
++       ((*(charpp) = const_cast<char*>(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \
+        PyBytes_AsStringAndSize(ob, (charpp), (sizep)))
+ #endif
+
+diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc
+index 5893533adf..31094b7e10 100644
+--- a/python/google/protobuf/pyext/message.cc
++++ b/python/google/protobuf/pyext/message.cc
+@@ -79,7 +79,7 @@
+     (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AsString(ob))
+   #define PyString_AsStringAndSize(ob, charpp, sizep) \
+     (PyUnicode_Check(ob)? \
+-       ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \
++       ((*(charpp) = const_cast<char*>(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \
+        PyBytes_AsStringAndSize(ob, (charpp), (sizep)))
+   #endif
+ #endif
+@@ -1529,7 +1529,7 @@ PyObject* HasField(CMessage* self, PyObject* arg) {
+     return NULL;
+   }
+ #else
+-  field_name = PyUnicode_AsUTF8AndSize(arg, &size);
++  field_name = const_cast<char*>(PyUnicode_AsUTF8AndSize(arg, &size));
+   if (!field_name) {
+     return NULL;
+   }