diff --git a/pythonforandroid/recipes/openssl/__init__.py b/pythonforandroid/recipes/openssl/__init__.py index 195fd3a2bc..aa01ac7f7f 100644 --- a/pythonforandroid/recipes/openssl/__init__.py +++ b/pythonforandroid/recipes/openssl/__init__.py @@ -60,7 +60,7 @@ def build_arch(self, arch): break shprint(sh.make, 'clean', _env=env) - self.install_libs(arch, 'libssl' + self.version + '.so', - 'libcrypto' + self.version + '.so') + self.install_libs(arch, 'libssl.a', 'libssl' + self.version + '.so', + 'libcrypto.a', 'libcrypto' + self.version + '.so') recipe = OpenSSLRecipe() diff --git a/pythonforandroid/recipes/python3crystax/__init__.py b/pythonforandroid/recipes/python3crystax/__init__.py index 60da4904f3..f461a4b9e6 100644 --- a/pythonforandroid/recipes/python3crystax/__init__.py +++ b/pythonforandroid/recipes/python3crystax/__init__.py @@ -1,76 +1,257 @@ - from pythonforandroid.recipe import TargetPythonRecipe from pythonforandroid.toolchain import shprint, current_directory, ArchARM from pythonforandroid.logger import info, error from pythonforandroid.util import ensure_dir, temp_directory from os.path import exists, join +import os import glob import sh +from sh import Command + +# This is the content of opensslconf.h taken from +# ndkdir/build/tools/build-target-openssl.sh +OPENSSLCONF = """#if defined(__ARM_ARCH_5TE__) +#include "opensslconf_armeabi.h" +#elif defined(__ARM_ARCH_7A__) && !defined(__ARM_PCS_VFP) +#include "opensslconf_armeabi_v7a.h" +#elif defined(__ARM_ARCH_7A__) && defined(__ARM_PCS_VFP) +#include "opensslconf_armeabi_v7a_hard.h" +#elif defined(__aarch64__) +#include "opensslconf_arm64_v8a.h" +#elif defined(__i386__) +#include "opensslconf_x86.h" +#elif defined(__x86_64__) +#include "opensslconf_x86_64.h" +#elif defined(__mips__) && !defined(__mips64) +#include "opensslconf_mips.h" +#elif defined(__mips__) && defined(__mips64) +#include "opensslconf_mips64.h" +#else +#error "Unsupported ABI" +#endif +""" +LATEST_FULL_VERSION = { + '3.5': '3.5.1', + '3.6': '3.6.4' +} + +def realpath(fname): + """ + Own implementation of os.realpath which may be broken in some python versions + Returns: the absolute path o + + """ + + if not os.path.islink(fname): + return os.path.abspath(fname) -prebuilt_download_locations = { - '3.6': ('https://github.com/inclement/crystax_python_builds/' - 'releases/download/0.1/crystax_python_3.6_armeabi_armeabi-v7a.tar.gz')} + abs_path = os.path.abspath(fname).split(os.sep)[:-1] + rel_path = os.readlink(fname) + + if os.path.abspath(rel_path) == rel_path: + return rel_path + + rel_path = rel_path.split(os.sep) + for folder in rel_path: + if folder == '..': + abs_path.pop() + else: + abs_path.append(folder) + return os.sep.join(abs_path) class Python3Recipe(TargetPythonRecipe): version = '3.5' url = '' name = 'python3crystax' - depends = ['hostpython3crystax'] + depends = ['hostpython3crystax'] conflicts = ['python2', 'python3'] from_crystax = True + def download_if_necessary(self): + if 'openssl' in self.ctx.recipe_build_order or self.version == '3.6': + full_version = LATEST_FULL_VERSION[self.version] + Python3Recipe.url = 'https://www.python.org/ftp/python/{0}.{1}.{2}/Python-{0}.{1}.{2}.tgz'.format(*full_version.split('.')) + super(Python3Recipe, self).download_if_necessary() + def get_dir_name(self): name = super(Python3Recipe, self).get_dir_name() name += '-version{}'.format(self.version) return name + def copy_include_dir(self, source, target): + ensure_dir(target) + for fname in os.listdir(source): + sh.ln('-sf', realpath(join(source, fname)), join(target, fname)) + + def _patch_dev_defaults(self, fp, target_ver): + for line in fp: + if 'OPENSSL_VERSIONS=' in line: + versions = line.split('"')[1].split(' ') + if versions[0] == target_ver: + raise ValueError('Patch not needed') + + if target_ver in versions: + versions.remove(target_ver) + + versions.insert(0, target_ver) + + yield 'OPENSSL_VERSIONS="{}"\n'.format(' '.join(versions)) + else: + yield line + + def patch_dev_defaults(self, ssl_recipe): + def_fname = join(self.ctx.ndk_dir, 'build', 'tools', 'dev-defaults.sh') + try: + with open(def_fname, 'r') as fp: + s = ''.join(self._patch_dev_defaults(fp, + str(ssl_recipe.version))) + with open(def_fname, 'w') as fp: + fp.write(s) + + except ValueError: + pass + + def check_for_sslso(self, ssl_recipe, arch): + # type: (Recipe, str) + dynlib_dir = join(self.ctx.ndk_dir, 'sources', 'python', self.version, + 'libs', arch.arch, 'modules') + + if os.path.exists(join(dynlib_dir, '_ssl.so')): + return 10, 'Shared object exists in ndk' + + # find out why _ssl.so is missing + + source_dir = join(self.ctx.ndk_dir, 'sources', 'openssl', ssl_recipe.version) + if not os.path.exists(source_dir): + return 0, 'Openssl version not present' + + # these two path checks are lifted straight from: + # crystax-ndk/build/tools/build-target-python.sh + if not os.path.exists(join(source_dir, 'Android.mk')): + return 1.1, 'Android.mk is missing in openssl source' + + include_dir = join(source_dir, 'include','openssl') + if not os.path.exists(join(include_dir, 'opensslconf.h')): + return 1.2, 'Openssl include dir missing' + + under_scored_arch = arch.arch.replace('-', '_') + if not os.path.lexists(join(include_dir, + 'opensslconf_{}.h'.format(under_scored_arch))): + return 1.3, 'Opensslconf arch header missing from include' + + + + # lastly a check to see whether shared objects for the correct arch + # is present in the ndk + if not os.path.exists(join(source_dir, 'libs', arch.arch)): + return 2, 'Openssl libs for this arch is missing in ndk' + + return 5, 'Ready to recompile python' + + def find_Android_mk(self): + openssl_dir = join(self.ctx.ndk_dir, 'sources', 'openssl') + for version in os.listdir(openssl_dir): + mk_path = join(openssl_dir, version, 'Android.mk') + if os.path.exists(mk_path): + return mk_path + + def prebuild_arch(self, arch): + super(Python3Recipe, self).prebuild_arch(arch) + if self.version == '3.6': + Python3Recipe.patches = ['patch_python3.6.patch'] + build_dir = self.get_build_dir(arch.arch) + shprint(sh.ln, '-sf', + realpath(join(build_dir, 'Lib/site-packages/README.txt')), + join(build_dir, 'Lib/site-packages/README')) + python_build_files = ['android.mk', 'config.c', 'interpreter.c'] + ndk_build_tools_python_dir = join(self.ctx.ndk_dir, 'build', 'tools', 'build-target-python') + for python_build_file in python_build_files: + shprint(sh.cp, join(ndk_build_tools_python_dir, python_build_file+'.3.5'), + join(ndk_build_tools_python_dir, python_build_file+'.3.6')) + ndk_sources_python_dir = join(self.ctx.ndk_dir, 'sources', 'python') + if not os.path.exists(join(ndk_sources_python_dir, '3.6')): + os.mkdir(join(ndk_sources_python_dir, '3.6')) + sh.sed('s#3.5#3.6#', + join(ndk_sources_python_dir, '3.5/Android.mk'), + _out=join(ndk_sources_python_dir, '3.6/Android.mk')) + def build_arch(self, arch): - # We don't have to actually build anything as CrystaX comes - # with the necessary modules. They are included by modifying - # the Android.mk in the jni folder. - - # If the Python version to be used is not prebuilt with the CrystaX - # NDK, we do have to download it. - - crystax_python_dir = join(self.ctx.ndk_dir, 'sources', 'python') - if not exists(join(crystax_python_dir, self.version)): - info(('The NDK does not have a prebuilt Python {}, trying ' - 'to obtain one.').format(self.version)) - - if self.version not in prebuilt_download_locations: - error(('No prebuilt version for Python {} could be found, ' - 'the built cannot continue.')) - exit(1) - - with temp_directory() as td: - self.download_file(prebuilt_download_locations[self.version], - join(td, 'downloaded_python')) - shprint(sh.tar, 'xf', join(td, 'downloaded_python'), - '--directory', crystax_python_dir) - - if not exists(join(crystax_python_dir, self.version)): - error(('Something went wrong, the directory at {} should ' - 'have been created but does not exist.').format( - join(crystax_python_dir, self.version))) - - if not exists(join( - crystax_python_dir, self.version, 'libs', arch.arch)): - error(('The prebuilt Python for version {} does not contain ' - 'binaries for your chosen architecture "{}".').format( - self.version, arch.arch)) - exit(1) - - # TODO: We should have an option to build a new Python. This - # would also allow linking to openssl and sqlite from CrystaX. + # If openssl is needed we may have to recompile cPython to get the + # ssl.py module working properly + if self.from_crystax and 'openssl' in self.ctx.recipe_build_order: + info('Openssl and crystax-python combination may require ' + 'recompilation of python...') + ssl_recipe = self.get_recipe('openssl', self.ctx) + stage, msg = self.check_for_sslso(ssl_recipe, arch) + stage = 0 if stage < 5 else stage + info(msg) + openssl_build_dir = ssl_recipe.get_build_dir(arch.arch) + openssl_ndk_dir = join(self.ctx.ndk_dir, 'sources', 'openssl', + ssl_recipe.version) + + if stage < 2: + info('Copying openssl headers and Android.mk to ndk') + ensure_dir(openssl_ndk_dir) + if stage < 1.2: + # copy include folder and Android.mk to ndk + mk_path = self.find_Android_mk() + if mk_path is None: + raise IOError('Android.mk file could not be found in ' + 'any versions in ndk->sources->openssl') + shprint(sh.cp, mk_path, openssl_ndk_dir) + + include_dir = join(openssl_build_dir, 'include') + if stage < 1.3: + ndk_include_dir = join(openssl_ndk_dir, 'include', 'openssl') + self.copy_include_dir(join(include_dir, 'openssl'), ndk_include_dir) + + target_conf = join(openssl_ndk_dir, 'include', 'openssl', + 'opensslconf.h') + shprint(sh.rm, '-f', target_conf) + # overwrite opensslconf.h + with open(target_conf, 'w') as fp: + fp.write(OPENSSLCONF) + + if stage < 1.4: + # move current conf to arch specific conf in ndk + under_scored_arch = arch.arch.replace('-', '_') + shprint(sh.ln, '-sf', + realpath(join(include_dir, 'openssl', 'opensslconf.h')), + join(openssl_ndk_dir, 'include', 'openssl', + 'opensslconf_{}.h'.format(under_scored_arch)) + ) + if stage < 3: + info('Copying openssl libs to ndk') + arch_ndk_lib = join(openssl_ndk_dir, 'libs', arch.arch) + ensure_dir(arch_ndk_lib) + shprint(sh.ln, '-sf', + realpath(join(openssl_build_dir, 'libcrypto{}.so'.format(ssl_recipe.version))), + join(openssl_build_dir, 'libcrypto.so')) + shprint(sh.ln, '-sf', + realpath(join(openssl_build_dir, 'libssl{}.so'.format(ssl_recipe.version))), + join(openssl_build_dir, 'libssl.so')) + libs = ['libcrypto.a', 'libcrypto.so', 'libssl.a', 'libssl.so'] + cmd = [join(openssl_build_dir, lib) for lib in libs] + [arch_ndk_lib] + shprint(sh.cp, '-f', *cmd) + + if stage < 10: + info('Recompiling python-crystax') + self.patch_dev_defaults(ssl_recipe) + build_script = join(self.ctx.ndk_dir, 'build', 'tools', + 'build-target-python.sh') + + shprint(Command(build_script), + '--ndk-dir={}'.format(self.ctx.ndk_dir), + '--abis={}'.format(arch.arch), + '-j5', '--verbose', + self.get_build_dir(arch.arch)) + + info('Extracting CrystaX python3 from NDK package') dirn = self.ctx.get_python_install_dir() ensure_dir(dirn) - - # Instead of using a locally built hostpython, we use the - # user's Python for now. They must have the right version - # available. Using e.g. pyenv makes this easy. self.ctx.hostpython = 'python{}'.format(self.version) recipe = Python3Recipe() diff --git a/pythonforandroid/recipes/python3crystax/patch_python3.6.patch b/pythonforandroid/recipes/python3crystax/patch_python3.6.patch new file mode 100644 index 0000000000..4402d1bd3e --- /dev/null +++ b/pythonforandroid/recipes/python3crystax/patch_python3.6.patch @@ -0,0 +1,89 @@ +diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c +--- a/Modules/expat/xmlparse.c ++++ b/Modules/expat/xmlparse.c +@@ -84,6 +84,8 @@ + # define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 + #endif + ++#define XML_POOR_ENTROPY 1 ++ + #if !defined(HAVE_GETRANDOM) && !defined(HAVE_SYSCALL_GETRANDOM) \ + && !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_ARC4RANDOM) \ + && !defined(XML_DEV_URANDOM) \ +diff --git a/Modules/getpath.c b/Modules/getpath.c +--- a/Modules/getpath.c ++++ b/Modules/getpath.c +@@ -101,8 +101,35 @@ + #endif + + +-#if !defined(PREFIX) || !defined(EXEC_PREFIX) || !defined(VERSION) || !defined(VPATH) +-#error "PREFIX, EXEC_PREFIX, VERSION, and VPATH must be constant defined" ++ /* These variables were set this way in old versions of Python, but ++ changed somewhere between 3.5.0 and 3.5.3. Here we just force ++ the old way again. A better solution would be to work out where ++ they should be defined, and make the CrystaX build scripts do ++ so. */ ++ ++/* #if !defined(PREFIX) || !defined(EXEC_PREFIX) || !defined(VERSION) || !defined(VPATH) */ ++/* #error "PREFIX, EXEC_PREFIX, VERSION, and VPATH must be constant defined" */ ++/* #endif */ ++ ++#ifndef VERSION ++#define VERSION "2.1" ++#endif ++ ++#ifndef VPATH ++#define VPATH "." ++#endif ++ ++#ifndef PREFIX ++# define PREFIX "/usr/local" ++#endif ++ ++#ifndef EXEC_PREFIX ++#define EXEC_PREFIX PREFIX ++#endif ++ ++#ifndef PYTHONPATH ++#define PYTHONPATH PREFIX "/lib/python" VERSION ":" \ ++ EXEC_PREFIX "/lib/python" VERSION "/lib-dynload" + #endif + + #ifndef LANDMARK +diff --git a/Modules/timemodule.c b/Modules/timemodule.c +--- a/Modules/timemodule.c ++++ b/Modules/timemodule.c +@@ -358,18 +358,20 @@ time_gmtime(PyObject *self, PyObject *args) + #endif + } + +-#ifndef HAVE_TIMEGM +-static time_t +-timegm(struct tm *p) +-{ +- /* XXX: the following implementation will not work for tm_year < 1970. +- but it is likely that platforms that don't have timegm do not support +- negative timestamps anyways. */ +- return p->tm_sec + p->tm_min*60 + p->tm_hour*3600 + p->tm_yday*86400 + +- (p->tm_year-70)*31536000 + ((p->tm_year-69)/4)*86400 - +- ((p->tm_year-1)/100)*86400 + ((p->tm_year+299)/400)*86400; +-} +-#endif ++/* In the Android build, HAVE_TIMEGM apparently should be defined but isn't. A better fix would be to work out why and fix that. */ ++ ++/* #ifndef HAVE_TIMEGM */ ++/* static time_t */ ++/* timegm(struct tm *p) */ ++/* { */ ++/* /\* XXX: the following implementation will not work for tm_year < 1970. */ ++/* but it is likely that platforms that don't have timegm do not support */ ++/* negative timestamps anyways. *\/ */ ++/* return p->tm_sec + p->tm_min*60 + p->tm_hour*3600 + p->tm_yday*86400 + */ ++/* (p->tm_year-70)*31536000 + ((p->tm_year-69)/4)*86400 - */ ++/* ((p->tm_year-1)/100)*86400 + ((p->tm_year+299)/400)*86400; */ ++/* } */ ++/* #endif */ + + PyDoc_STRVAR(gmtime_doc, + "gmtime([seconds]) -> (tm_year, tm_mon, tm_mday, tm_hour, tm_min,\n\