From ea461ff8c001a1f619bfb2d0c29a8ba51c2d939d Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Thu, 29 Aug 2019 22:50:18 +0200 Subject: [PATCH] [LIBS - PART II] Part II of NDK r19 migration - Initial STL lib migration (#1947) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [recipe-stl] Add android's STL lib support to `Recipe` To allow us to refactor some common operations that we use in our recipes that depends on android's STL library. Note: This commit will allow us to begin the migration to `c++_shared`. This is a must when we move to android's NDK r19+, because as for android NDK >= 18 is the only one supported STL library. * [recipe-stl] Make CppCompiledComponentsPythonRecipe use Recipe's STL support * [recipe-stl] Make icu a library recipe with STL support (rework) Also done here:   - Remove hardcoded version in url   - Disable versioned shared libraries   - Make it to be build as a shared libraries (instead of static)   - Disable the build of static libraries (because we build them as shared ones, so we avoid to link with them without our consents)   - Shorten long lines to be pep8's friendly   - Remove icu from ci/constants   - Remove `RuntimeError` about the need to use NDK api <= 19 (because that is not the case anymore)   - consider host's number of cpu to perform the build * [recipe-stl] Rework pyicu recipe to match latest icu changes Also done here:   - Remove icu.patch because now we don't have the version in our icu libraries   - Shorten long lines to be pep8's friendly * [tests] Add tests for recipe with STL support * [tests] Add tests for icu recipe * [tests] Add tests for pyicu recipe --- ci/constants.py | 1 - pythonforandroid/recipe.py | 90 ++++++---- pythonforandroid/recipes/icu/__init__.py | 123 +++++-------- .../recipes/icu/disable-libs-version.patch | 66 +++++++ pythonforandroid/recipes/pyicu/__init__.py | 55 ++---- pythonforandroid/recipes/pyicu/icu.patch | 19 -- tests/recipes/test_icu.py | 167 ++++++++++++++++++ tests/recipes/test_pyicu.py | 88 +++++++++ tests/test_recipe.py | 122 +++++++++++++ 9 files changed, 563 insertions(+), 168 deletions(-) create mode 100644 pythonforandroid/recipes/icu/disable-libs-version.patch delete mode 100644 pythonforandroid/recipes/pyicu/icu.patch create mode 100644 tests/recipes/test_icu.py create mode 100644 tests/recipes/test_pyicu.py diff --git a/ci/constants.py b/ci/constants.py index c5ab61099d..6699d24cd3 100644 --- a/ci/constants.py +++ b/ci/constants.py @@ -65,7 +65,6 @@ class TargetPython(Enum): # IndexError: list index out of range 'secp256k1', 'ffpyplayer', - 'icu', # requires `libpq-dev` system dependency e.g. for `pg_config` binary 'psycopg2', 'protobuf_cpp', diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 0edca4a514..3d814d5c4e 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -125,6 +125,46 @@ class Recipe(with_metaclass(RecipeMeta)): path: `'.', None or ''` """ + need_stl_shared = False + '''Some libraries or python packages may need to be linked with android's + stl. We can automatically do this for any recipe if we set this property to + `True`''' + + stl_lib_name = 'c++_shared' + ''' + The default STL shared lib to use: `c++_shared`. + + .. note:: Android NDK version > 17 only supports 'c++_shared', because + starting from NDK r18 the `gnustl_shared` lib has been deprecated. + ''' + + stl_lib_source = '{ctx.ndk_dir}/sources/cxx-stl/llvm-libc++' + ''' + The source directory of the selected stl lib, defined in property + `stl_lib_name` + ''' + + @property + def stl_include_dir(self): + return join(self.stl_lib_source.format(ctx=self.ctx), 'include') + + def get_stl_lib_dir(self, arch): + return join( + self.stl_lib_source.format(ctx=self.ctx), 'libs', arch.arch + ) + + def get_stl_library(self, arch): + return join( + self.get_stl_lib_dir(arch), + 'lib{name}.so'.format(name=self.stl_lib_name), + ) + + def install_stl_lib(self, arch): + if not self.ctx.has_lib( + arch.arch, 'lib{name}.so'.format(name=self.stl_lib_name) + ): + self.install_libs(arch, self.get_stl_library(arch)) + @property def version(self): key = 'VERSION_' + self.name @@ -454,7 +494,22 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): """ if arch is None: arch = self.filtered_archs[0] - return arch.get_env(with_flags_in_cc=with_flags_in_cc) + env = arch.get_env(with_flags_in_cc=with_flags_in_cc) + + if self.need_stl_shared: + env['CPPFLAGS'] = env.get('CPPFLAGS', '') + env['CPPFLAGS'] += ' -I{}'.format(self.stl_include_dir) + + env['CXXFLAGS'] = env['CFLAGS'] + ' -frtti -fexceptions' + + if with_flags_in_cc: + env['CXX'] += ' -frtti -fexceptions' + + env['LDFLAGS'] += ' -L{}'.format(self.get_stl_lib_dir(arch)) + env['LIBS'] = env.get('LIBS', '') + " -l{}".format( + self.stl_lib_name + ) + return env def prebuild_arch(self, arch): '''Run any pre-build tasks for the Recipe. By default, this checks if @@ -538,6 +593,9 @@ def postbuild_arch(self, arch): if hasattr(self, postbuild): getattr(self, postbuild)() + if self.need_stl_shared: + self.install_stl_lib(arch) + def prepare_build_dir(self, arch): '''Copies the recipe data into a build dir for the given arch. By default, this unpacks a downloaded recipe. You should override @@ -982,35 +1040,7 @@ def rebuild_compiled_components(self, arch, env): class CppCompiledComponentsPythonRecipe(CompiledComponentsPythonRecipe): """ Extensions that require the cxx-stl """ call_hostpython_via_targetpython = False - - def get_recipe_env(self, arch): - env = super(CppCompiledComponentsPythonRecipe, self).get_recipe_env(arch) - keys = dict( - ctx=self.ctx, - arch=arch, - arch_noeabi=arch.arch.replace('eabi', '') - ) - env['LDSHARED'] = env['CC'] + ' -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions' - env['CFLAGS'] += ( - " -I{ctx.ndk_dir}/platforms/android-{ctx.android_api}/arch-{arch_noeabi}/usr/include" + - " -I{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/include" + - " -I{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/include").format(**keys) - env['CXXFLAGS'] = env['CFLAGS'] + ' -frtti -fexceptions' - env['LDFLAGS'] += ( - " -L{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}" + - " -lgnustl_shared").format(**keys) - - return env - - def build_compiled_components(self, arch): - super(CppCompiledComponentsPythonRecipe, self).build_compiled_components(arch) - - # Copy libgnustl_shared.so - with current_directory(self.get_build_dir(arch.arch)): - sh.cp( - "{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/libgnustl_shared.so".format(ctx=self.ctx, arch=arch), - self.ctx.get_libs_dir(arch.arch) - ) + need_stl_shared = True class CythonRecipe(PythonRecipe): diff --git a/pythonforandroid/recipes/icu/__init__.py b/pythonforandroid/recipes/icu/__init__.py index 4bb2de0c99..43c5ac9eb8 100644 --- a/pythonforandroid/recipes/icu/__init__.py +++ b/pythonforandroid/recipes/icu/__init__.py @@ -1,33 +1,54 @@ import sh import os -from os.path import join, isdir -from pythonforandroid.recipe import NDKRecipe +from os.path import join, isdir, exists +from multiprocessing import cpu_count +from pythonforandroid.recipe import Recipe from pythonforandroid.toolchain import shprint from pythonforandroid.util import current_directory, ensure_dir -class ICURecipe(NDKRecipe): +class ICURecipe(Recipe): name = 'icu4c' version = '57.1' - url = 'http://download.icu-project.org/files/icu4c/57.1/icu4c-57_1-src.tgz' + major_version = version.split('.')[0] + url = ('http://download.icu-project.org/files/icu4c/' + '{version}/icu4c-{version_underscore}-src.tgz') depends = [('hostpython2', 'hostpython3')] # installs in python - generated_libraries = [ - 'libicui18n.so', 'libicuuc.so', 'libicudata.so', 'libicule.so'] - - def get_lib_dir(self, arch): - lib_dir = join(self.ctx.get_python_install_dir(), "lib") - ensure_dir(lib_dir) - return lib_dir - - def prepare_build_dir(self, arch): - if self.ctx.android_api > 19: - # greater versions do not have /usr/include/sys/exec_elf.h - raise RuntimeError("icu needs an android api <= 19") - - super(ICURecipe, self).prepare_build_dir(arch) - - def build_arch(self, arch, *extra_args): + patches = ['disable-libs-version.patch'] + + built_libraries = { + 'libicui18n{}.so'.format(major_version): 'build_icu_android/lib', + 'libicuuc{}.so'.format(major_version): 'build_icu_android/lib', + 'libicudata{}.so'.format(major_version): 'build_icu_android/lib', + 'libicule{}.so'.format(major_version): 'build_icu_android/lib', + 'libicuio{}.so'.format(major_version): 'build_icu_android/lib', + 'libicutu{}.so'.format(major_version): 'build_icu_android/lib', + 'libiculx{}.so'.format(major_version): 'build_icu_android/lib', + } + need_stl_shared = True + + @property + def versioned_url(self): + if self.url is None: + return None + return self.url.format( + version=self.version, + version_underscore=self.version.replace('.', '_')) + + def get_recipe_dir(self): + """ + .. note:: We need to overwrite `Recipe.get_recipe_dir` due to the + mismatch name between the recipe's folder (icu) and the value + of `ICURecipe.name` (icu4c). + """ + if self.ctx.local_recipes is not None: + local_recipe_dir = join(self.ctx.local_recipes, 'icu') + if exists(local_recipe_dir): + return local_recipe_dir + return join(self.ctx.root_dir, 'recipes', 'icu') + + def build_arch(self, arch): env = self.get_recipe_env(arch).copy() build_root = self.get_build_dir(arch.arch) @@ -60,11 +81,11 @@ def make_build_dest(dest): "--prefix="+icu_build, "--enable-extras=no", "--enable-strict=no", - "--enable-static", + "--enable-static=no", "--enable-tests=no", "--enable-samples=no", _env=host_env) - shprint(sh.make, "-j5", _env=host_env) + shprint(sh.make, "-j", str(cpu_count()), _env=host_env) shprint(sh.make, "install", _env=host_env) build_android, exists = make_build_dest("build_icu_android") @@ -72,62 +93,23 @@ def make_build_dest(dest): configure = sh.Command(join(build_root, "source", "configure")) - include = ( - " -I{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/include/" - " -I{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/" - "{arch}/include") - include = include.format(ndk=self.ctx.ndk_dir, - version=env["TOOLCHAIN_VERSION"], - arch=arch.arch) - env["CPPFLAGS"] = env["CXXFLAGS"] + " " - env["CPPFLAGS"] += host_env["CPPFLAGS"] - env["CPPFLAGS"] += include - - lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}" - lib = lib.format(ndk=self.ctx.ndk_dir, - version=env["TOOLCHAIN_VERSION"], - arch=arch.arch) - env["LDFLAGS"] += " -lgnustl_shared -L"+lib - - env.pop("CFLAGS", None) - env.pop("CXXFLAGS", None) - with current_directory(build_android): shprint( configure, "--with-cross-build="+build_linux, "--enable-extras=no", "--enable-strict=no", - "--enable-static", + "--enable-static=no", "--enable-tests=no", "--enable-samples=no", "--host="+env["TOOLCHAIN_PREFIX"], "--prefix="+icu_build, _env=env) - shprint(sh.make, "-j5", _env=env) + shprint(sh.make, "-j", str(cpu_count()), _env=env) shprint(sh.make, "install", _env=env) - self.copy_files(arch) - - def copy_files(self, arch): - env = self.get_recipe_env(arch) - - lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}" - lib = lib.format(ndk=self.ctx.ndk_dir, - version=env["TOOLCHAIN_VERSION"], - arch=arch.arch) - stl_lib = join(lib, "libgnustl_shared.so") - dst_dir = join(self.ctx.get_site_packages_dir(), "..", "lib-dynload") - shprint(sh.cp, stl_lib, dst_dir) - - src_lib = join(self.get_build_dir(arch.arch), "icu_build", "lib") - dst_lib = self.get_lib_dir(arch) - - src_suffix = "." + self.version - dst_suffix = "." + self.version.split(".")[0] # main version - for lib in self.generated_libraries: - shprint(sh.cp, join(src_lib, lib+src_suffix), - join(dst_lib, lib+dst_suffix)) + def install_libraries(self, arch): + super(ICURecipe, self).install_libraries(arch) src_include = join( self.get_build_dir(arch.arch), "icu_build", "include") @@ -137,16 +119,5 @@ def copy_files(self, arch): shprint(sh.cp, "-r", join(src_include, "layout"), dst_include) shprint(sh.cp, "-r", join(src_include, "unicode"), dst_include) - # copy stl library - lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}" - lib = lib.format(ndk=self.ctx.ndk_dir, - version=env["TOOLCHAIN_VERSION"], - arch=arch.arch) - stl_lib = join(lib, "libgnustl_shared.so") - - dst_dir = join(self.ctx.get_python_install_dir(), "lib") - ensure_dir(dst_dir) - shprint(sh.cp, stl_lib, dst_dir) - recipe = ICURecipe() diff --git a/pythonforandroid/recipes/icu/disable-libs-version.patch b/pythonforandroid/recipes/icu/disable-libs-version.patch new file mode 100644 index 0000000000..872abe01e4 --- /dev/null +++ b/pythonforandroid/recipes/icu/disable-libs-version.patch @@ -0,0 +1,66 @@ +diff -aur icu4c-org/source/config/Makefile.inc.in icu4c/source/config/Makefile.inc.in +--- icu/source/config/Makefile.inc.in.orig 2016-03-23 21:50:50.000000000 +0100 ++++ icu/source/config/Makefile.inc.in 2019-02-15 17:59:28.331749766 +0100 +@@ -142,8 +142,8 @@ + LDLIBRARYPATH_ENVVAR = LD_LIBRARY_PATH + + # Versioned target for a shared library +-FINAL_SO_TARGET = $(SO_TARGET).$(SO_TARGET_VERSION) +-MIDDLE_SO_TARGET = $(SO_TARGET).$(SO_TARGET_VERSION_MAJOR) ++FINAL_SO_TARGET = $(SO_TARGET).$(SO_TARGET_VERSION) ++MIDDLE_SO_TARGET = $(SO_TARGET) + + # Access to important ICU tools. + # Use as follows: $(INVOKE) $(GENRB) arguments .. +diff -aur icu4c-org/source/config/mh-linux icu4c/source/config/mh-linux +--- icu4c-org/source/config/mh-linux 2017-07-05 13:23:06.000000000 +0200 ++++ icu4c/source/config/mh-linux 2017-07-06 14:02:52.275016858 +0200 +@@ -24,9 +24,17 @@ + + ## Compiler switch to embed a library name + # The initial tab in the next line is to prevent icu-config from reading it. +- LD_SONAME = -Wl,-soname -Wl,$(notdir $(MIDDLE_SO_TARGET)) ++ LD_SONAME = -Wl,-soname -Wl,$(notdir $(SO_TARGET)) ++ DATA_STUBNAME = data$(SO_TARGET_VERSION_MAJOR) ++ COMMON_STUBNAME = uc$(SO_TARGET_VERSION_MAJOR) ++ I18N_STUBNAME = i18n$(SO_TARGET_VERSION_MAJOR) ++ LAYOUT_STUBNAME = le$(SO_TARGET_VERSION_MAJOR) ++ LAYOUTEX_STUBNAME = lx$(SO_TARGET_VERSION_MAJOR) ++ IO_STUBNAME = io$(SO_TARGET_VERSION_MAJOR) ++ TOOLUTIL_STUBNAME = tu$(SO_TARGET_VERSION_MAJOR) ++ CTESTFW_STUBNAME = test$(SO_TARGET_VERSION_MAJOR) + #SH# # We can't depend on MIDDLE_SO_TARGET being set. +-#SH# LD_SONAME= ++#SH# LD_SONAME=$(SO_TARGET) + + ## Shared library options + LD_SOOPTIONS= -Wl,-Bsymbolic +@@ -64,10 +64,10 @@ + + ## Versioned libraries rules + +-%.$(SO).$(SO_TARGET_VERSION_MAJOR): %.$(SO).$(SO_TARGET_VERSION) +- $(RM) $@ && ln -s ${