[LIBS - PART II] Part II of NDK r19 migration - Initial STL lib migra…

…tion (kivy#1947)

* [recipe-stl] Add android's STL lib support to `Recipe`

To allow us to refactor some common operations that we use in our recipes that depends on android's STL library.

Note: This commit will allow us to begin the migration to `c++_shared`. This is a must when we move to android's NDK r19+, because as for android NDK >= 18 is the only one supported STL library.

* [recipe-stl] Make CppCompiledComponentsPythonRecipe use Recipe's STL support

* [recipe-stl] Make icu a library recipe with STL support (rework)

Also done here:
  - Remove hardcoded version in url
  - Disable versioned shared libraries
  - Make it to be build as a shared libraries (instead of static)
  - Disable the build of static libraries (because we build them as shared ones, so we avoid to link with them without our consents)
  - Shorten long lines to be pep8's friendly
  - Remove icu from ci/constants
  - Remove `RuntimeError` about the need to use NDK api <= 19 (because that is not the case anymore)
  - consider host's number of cpu to perform the build

* [recipe-stl] Rework pyicu recipe to match latest icu changes

Also done here:
  - Remove icu.patch because now we don't have the version in our icu libraries
  - Shorten long lines to be pep8's friendly

* [tests] Add tests for recipe with STL support

* [tests] Add tests for icu recipe

* [tests] Add tests for pyicu recipe

ci/constants.py

@@ -65,7 +65,6 @@ class TargetPython(Enum):
     # IndexError: list index out of range
     'secp256k1',
     'ffpyplayer',
-    'icu',
     # requires `libpq-dev` system dependency e.g. for `pg_config` binary
     'psycopg2',
     'protobuf_cpp',

pythonforandroid/recipe.py

@@ -125,6 +125,46 @@ class Recipe(with_metaclass(RecipeMeta)):
               path: `'.', None or ''`
     """
 
+    need_stl_shared = False
+    '''Some libraries or python packages may need to be linked with android's
+    stl. We can automatically do this for any recipe if we set this property to
+    `True`'''
+
+    stl_lib_name = 'c++_shared'
+    '''
+    The default STL shared lib to use: `c++_shared`.
+
+    .. note:: Android NDK version > 17 only supports 'c++_shared', because
+        starting from NDK r18 the `gnustl_shared` lib has been deprecated.
+    '''
+
+    stl_lib_source = '{ctx.ndk_dir}/sources/cxx-stl/llvm-libc++'
+    '''
+    The source directory of the selected stl lib, defined in property
+    `stl_lib_name`
+    '''
+
+    @property
+    def stl_include_dir(self):
+        return join(self.stl_lib_source.format(ctx=self.ctx), 'include')
+
+    def get_stl_lib_dir(self, arch):
+        return join(
+            self.stl_lib_source.format(ctx=self.ctx), 'libs', arch.arch
+        )
+
+    def get_stl_library(self, arch):
+        return join(
+            self.get_stl_lib_dir(arch),
+            'lib{name}.so'.format(name=self.stl_lib_name),
+        )
+
+    def install_stl_lib(self, arch):
+        if not self.ctx.has_lib(
+            arch.arch, 'lib{name}.so'.format(name=self.stl_lib_name)
+        ):
+            self.install_libs(arch, self.get_stl_library(arch))
+
     @property
     def version(self):
         key = 'VERSION_' + self.name
@@ -454,7 +494,22 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         """
         if arch is None:
             arch = self.filtered_archs[0]
-        return arch.get_env(with_flags_in_cc=with_flags_in_cc)
+        env = arch.get_env(with_flags_in_cc=with_flags_in_cc)
+
+        if self.need_stl_shared:
+            env['CPPFLAGS'] = env.get('CPPFLAGS', '')
+            env['CPPFLAGS'] += ' -I{}'.format(self.stl_include_dir)
+
+            env['CXXFLAGS'] = env['CFLAGS'] + ' -frtti -fexceptions'
+
+            if with_flags_in_cc:
+                env['CXX'] += ' -frtti -fexceptions'
+
+            env['LDFLAGS'] += ' -L{}'.format(self.get_stl_lib_dir(arch))
+            env['LIBS'] = env.get('LIBS', '') + " -l{}".format(
+                self.stl_lib_name
+            )
+        return env
 
     def prebuild_arch(self, arch):
         '''Run any pre-build tasks for the Recipe. By default, this checks if
@@ -538,6 +593,9 @@ def postbuild_arch(self, arch):
         if hasattr(self, postbuild):
             getattr(self, postbuild)()
 
+        if self.need_stl_shared:
+            self.install_stl_lib(arch)
+
     def prepare_build_dir(self, arch):
         '''Copies the recipe data into a build dir for the given arch. By
         default, this unpacks a downloaded recipe. You should override
@@ -982,35 +1040,7 @@ def rebuild_compiled_components(self, arch, env):
 class CppCompiledComponentsPythonRecipe(CompiledComponentsPythonRecipe):
     """ Extensions that require the cxx-stl """
     call_hostpython_via_targetpython = False
-
-    def get_recipe_env(self, arch):
-        env = super(CppCompiledComponentsPythonRecipe, self).get_recipe_env(arch)
-        keys = dict(
-            ctx=self.ctx,
-            arch=arch,
-            arch_noeabi=arch.arch.replace('eabi', '')
-        )
-        env['LDSHARED'] = env['CC'] + ' -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions'
-        env['CFLAGS'] += (
-            " -I{ctx.ndk_dir}/platforms/android-{ctx.android_api}/arch-{arch_noeabi}/usr/include" +
-            " -I{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/include" +
-            " -I{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/include").format(**keys)
-        env['CXXFLAGS'] = env['CFLAGS'] + ' -frtti -fexceptions'
-        env['LDFLAGS'] += (
-            " -L{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}" +
-            " -lgnustl_shared").format(**keys)
-
-        return env
-
-    def build_compiled_components(self, arch):
-        super(CppCompiledComponentsPythonRecipe, self).build_compiled_components(arch)
-
-        # Copy libgnustl_shared.so
-        with current_directory(self.get_build_dir(arch.arch)):
-            sh.cp(
-                "{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/libgnustl_shared.so".format(ctx=self.ctx, arch=arch),
-                self.ctx.get_libs_dir(arch.arch)
-            )
+    need_stl_shared = True
 
 
 class CythonRecipe(PythonRecipe):

pythonforandroid/recipes/icu/__init__.py

@@ -1,33 +1,54 @@
 import sh
 import os
-from os.path import join, isdir
-from pythonforandroid.recipe import NDKRecipe
+from os.path import join, isdir, exists
+from multiprocessing import cpu_count
+from pythonforandroid.recipe import Recipe
 from pythonforandroid.toolchain import shprint
 from pythonforandroid.util import current_directory, ensure_dir
 
 
-class ICURecipe(NDKRecipe):
+class ICURecipe(Recipe):
     name = 'icu4c'
     version = '57.1'
-    url = 'http://download.icu-project.org/files/icu4c/57.1/icu4c-57_1-src.tgz'
+    major_version = version.split('.')[0]
+    url = ('http://download.icu-project.org/files/icu4c/'
+           '{version}/icu4c-{version_underscore}-src.tgz')
 
     depends = [('hostpython2', 'hostpython3')]  # installs in python
-    generated_libraries = [
-        'libicui18n.so', 'libicuuc.so', 'libicudata.so', 'libicule.so']
-
-    def get_lib_dir(self, arch):
-        lib_dir = join(self.ctx.get_python_install_dir(), "lib")
-        ensure_dir(lib_dir)
-        return lib_dir
-
-    def prepare_build_dir(self, arch):
-        if self.ctx.android_api > 19:
-            # greater versions do not have /usr/include/sys/exec_elf.h
-            raise RuntimeError("icu needs an android api <= 19")
-
-        super(ICURecipe, self).prepare_build_dir(arch)
-
-    def build_arch(self, arch, *extra_args):
+    patches = ['disable-libs-version.patch']
+
+    built_libraries = {
+        'libicui18n{}.so'.format(major_version): 'build_icu_android/lib',
+        'libicuuc{}.so'.format(major_version): 'build_icu_android/lib',
+        'libicudata{}.so'.format(major_version): 'build_icu_android/lib',
+        'libicule{}.so'.format(major_version): 'build_icu_android/lib',
+        'libicuio{}.so'.format(major_version): 'build_icu_android/lib',
+        'libicutu{}.so'.format(major_version): 'build_icu_android/lib',
+        'libiculx{}.so'.format(major_version): 'build_icu_android/lib',
+    }
+    need_stl_shared = True
+
+    @property
+    def versioned_url(self):
+        if self.url is None:
+            return None
+        return self.url.format(
+            version=self.version,
+            version_underscore=self.version.replace('.', '_'))
+
+    def get_recipe_dir(self):
+        """
+        .. note:: We need to overwrite `Recipe.get_recipe_dir` due to the
+                  mismatch name between the recipe's folder (icu) and the value
+                  of `ICURecipe.name` (icu4c).
+        """
+        if self.ctx.local_recipes is not None:
+            local_recipe_dir = join(self.ctx.local_recipes, 'icu')
+            if exists(local_recipe_dir):
+                return local_recipe_dir
+        return join(self.ctx.root_dir, 'recipes', 'icu')
+
+    def build_arch(self, arch):
         env = self.get_recipe_env(arch).copy()
         build_root = self.get_build_dir(arch.arch)
 
@@ -60,74 +81,35 @@ def make_build_dest(dest):
                     "--prefix="+icu_build,
                     "--enable-extras=no",
                     "--enable-strict=no",
-                    "--enable-static",
+                    "--enable-static=no",
                     "--enable-tests=no",
                     "--enable-samples=no",
                     _env=host_env)
-                shprint(sh.make, "-j5", _env=host_env)
+                shprint(sh.make, "-j", str(cpu_count()), _env=host_env)
                 shprint(sh.make, "install", _env=host_env)
 
         build_android, exists = make_build_dest("build_icu_android")
         if not exists:
 
             configure = sh.Command(join(build_root, "source", "configure"))
 
-            include = (
-                " -I{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/include/"
-                " -I{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/"
-                "{arch}/include")
-            include = include.format(ndk=self.ctx.ndk_dir,
-                                     version=env["TOOLCHAIN_VERSION"],
-                                     arch=arch.arch)
-            env["CPPFLAGS"] = env["CXXFLAGS"] + " "
-            env["CPPFLAGS"] += host_env["CPPFLAGS"]
-            env["CPPFLAGS"] += include
-
-            lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}"
-            lib = lib.format(ndk=self.ctx.ndk_dir,
-                             version=env["TOOLCHAIN_VERSION"],
-                             arch=arch.arch)
-            env["LDFLAGS"] += " -lgnustl_shared -L"+lib
-
-            env.pop("CFLAGS", None)
-            env.pop("CXXFLAGS", None)
-
             with current_directory(build_android):
                 shprint(
                     configure,
                     "--with-cross-build="+build_linux,
                     "--enable-extras=no",
                     "--enable-strict=no",
-                    "--enable-static",
+                    "--enable-static=no",
                     "--enable-tests=no",
                     "--enable-samples=no",
                     "--host="+env["TOOLCHAIN_PREFIX"],
                     "--prefix="+icu_build,
                     _env=env)
-                shprint(sh.make, "-j5", _env=env)
+                shprint(sh.make, "-j", str(cpu_count()), _env=env)
                 shprint(sh.make, "install", _env=env)
 
-        self.copy_files(arch)
-
-    def copy_files(self, arch):
-        env = self.get_recipe_env(arch)
-
-        lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}"
-        lib = lib.format(ndk=self.ctx.ndk_dir,
-                         version=env["TOOLCHAIN_VERSION"],
-                         arch=arch.arch)
-        stl_lib = join(lib, "libgnustl_shared.so")
-        dst_dir = join(self.ctx.get_site_packages_dir(), "..", "lib-dynload")
-        shprint(sh.cp, stl_lib, dst_dir)
-
-        src_lib = join(self.get_build_dir(arch.arch), "icu_build", "lib")
-        dst_lib = self.get_lib_dir(arch)
-
-        src_suffix = "." + self.version
-        dst_suffix = "." + self.version.split(".")[0]  # main version
-        for lib in self.generated_libraries:
-            shprint(sh.cp, join(src_lib, lib+src_suffix),
-                    join(dst_lib, lib+dst_suffix))
+    def install_libraries(self, arch):
+        super(ICURecipe, self).install_libraries(arch)
 
         src_include = join(
             self.get_build_dir(arch.arch), "icu_build", "include")
@@ -137,16 +119,5 @@ def copy_files(self, arch):
         shprint(sh.cp, "-r", join(src_include, "layout"), dst_include)
         shprint(sh.cp, "-r", join(src_include, "unicode"), dst_include)
 
-        # copy stl library
-        lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}"
-        lib = lib.format(ndk=self.ctx.ndk_dir,
-                         version=env["TOOLCHAIN_VERSION"],
-                         arch=arch.arch)
-        stl_lib = join(lib, "libgnustl_shared.so")
-
-        dst_dir = join(self.ctx.get_python_install_dir(), "lib")
-        ensure_dir(dst_dir)
-        shprint(sh.cp, stl_lib, dst_dir)
-
 
 recipe = ICURecipe()

pythonforandroid/recipes/icu/disable-libs-version.patch

@@ -0,0 +1,66 @@
+diff -aur icu4c-org/source/config/Makefile.inc.in icu4c/source/config/Makefile.inc.in
+--- icu/source/config/Makefile.inc.in.orig	2016-03-23 21:50:50.000000000 +0100
++++ icu/source/config/Makefile.inc.in	2019-02-15 17:59:28.331749766 +0100
+@@ -142,8 +142,8 @@
+ LDLIBRARYPATH_ENVVAR = LD_LIBRARY_PATH
+
+ # Versioned target for a shared library
+-FINAL_SO_TARGET = $(SO_TARGET).$(SO_TARGET_VERSION)
+-MIDDLE_SO_TARGET = $(SO_TARGET).$(SO_TARGET_VERSION_MAJOR)
++FINAL_SO_TARGET =  $(SO_TARGET).$(SO_TARGET_VERSION)
++MIDDLE_SO_TARGET = $(SO_TARGET)
+
+ # Access to important ICU tools.
+ # Use as follows:  $(INVOKE) $(GENRB) arguments ..
+diff -aur icu4c-org/source/config/mh-linux icu4c/source/config/mh-linux
+--- icu4c-org/source/config/mh-linux	2017-07-05 13:23:06.000000000 +0200
++++ icu4c/source/config/mh-linux	2017-07-06 14:02:52.275016858 +0200
+@@ -24,9 +24,17 @@
+
+ ## Compiler switch to embed a library name
+ # The initial tab in the next line is to prevent icu-config from reading it.
+-	LD_SONAME = -Wl,-soname -Wl,$(notdir $(MIDDLE_SO_TARGET))
++	LD_SONAME = -Wl,-soname -Wl,$(notdir $(SO_TARGET))
++	DATA_STUBNAME = data$(SO_TARGET_VERSION_MAJOR)
++	COMMON_STUBNAME = uc$(SO_TARGET_VERSION_MAJOR)
++	I18N_STUBNAME = i18n$(SO_TARGET_VERSION_MAJOR)
++	LAYOUT_STUBNAME = le$(SO_TARGET_VERSION_MAJOR)
++	LAYOUTEX_STUBNAME = lx$(SO_TARGET_VERSION_MAJOR)
++	IO_STUBNAME = io$(SO_TARGET_VERSION_MAJOR)
++	TOOLUTIL_STUBNAME = tu$(SO_TARGET_VERSION_MAJOR)
++	CTESTFW_STUBNAME = test$(SO_TARGET_VERSION_MAJOR)
+ #SH# # We can't depend on MIDDLE_SO_TARGET being set.
+-#SH# LD_SONAME=
++#SH# LD_SONAME=$(SO_TARGET)
+
+ ## Shared library options
+ LD_SOOPTIONS= -Wl,-Bsymbolic
+@@ -64,10 +64,10 @@
+
+ ## Versioned libraries rules
+
+-%.$(SO).$(SO_TARGET_VERSION_MAJOR): %.$(SO).$(SO_TARGET_VERSION)
+-	$(RM) $@ && ln -s ${<F} $@
+-%.$(SO): %.$(SO).$(SO_TARGET_VERSION_MAJOR)
+-	$(RM) $@ && ln -s ${*F}.$(SO).$(SO_TARGET_VERSION) $@
++%.$(SO).$(SO_TARGET_VERSION_MAJOR): %.$(SO).$(SO_TARGET_VERSION)
++	$(RM) $@ && ln -s ${<F} $@
++%.$(SO): %.$(SO).$(SO_TARGET_VERSION_MAJOR)
++	$(RM) $@ && ln -s ${*F}.$(SO).$(SO_TARGET_VERSION) $@
+
+ ##  Bind internal references
+
+diff -aur icu4c-org/source/data/Makefile.in icu4c/source/data/Makefile.in
+--- icu4c-org/source/data/Makefile.in	2017-07-05 13:23:06.000000000 +0200
++++ icu4c/source/data/Makefile.in	2017-07-06 14:05:31.607995855 +0200
+@@ -24,9 +24,9 @@
+ ifeq ($(PKGDATA_OPTS),)
+ PKGDATA_OPTS = -O $(top_builddir)/data/icupkg.inc
+ endif
+-ifeq ($(PKGDATA_VERSIONING),)
+-PKGDATA_VERSIONING = -r $(SO_TARGET_VERSION)
+-endif
++#ifeq ($(PKGDATA_VERSIONING),)
++#PKGDATA_VERSIONING = -r $(SO_TARGET_VERSION)
++#endif
+