diff --git a/.gitignore b/.gitignore index 6a8964aa0f..05196a8acb 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,19 @@ __pycache__/ #idea/pycharm .idea/ -.tox \ No newline at end of file + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +coverage.xml +*.cover +.pytest_cache/ + +# testapp's build folder +testapps/build/ + +# Dolphin (the KDE file manager autogenerates the file `.directory`) +.directory diff --git a/Dockerfile.py2 b/Dockerfile.py2 index 05f956aa22..5c9202c6df 100644 --- a/Dockerfile.py2 +++ b/Dockerfile.py2 @@ -58,8 +58,8 @@ RUN ${RETRY} curl --location --progress-bar --insecure \ ENV ANDROID_SDK_HOME="${ANDROID_HOME}/android-sdk" # get the latest version from https://developer.android.com/studio/index.html -ENV ANDROID_SDK_TOOLS_VERSION="3859397" -ENV ANDROID_SDK_BUILD_TOOLS_VERSION="26.0.2" +ENV ANDROID_SDK_TOOLS_VERSION="4333796" +ENV ANDROID_SDK_BUILD_TOOLS_VERSION="28.0.2" ENV ANDROID_SDK_TOOLS_ARCHIVE="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip" ENV ANDROID_SDK_TOOLS_DL_URL="https://dl.google.com/android/repository/${ANDROID_SDK_TOOLS_ARCHIVE}" @@ -76,16 +76,14 @@ RUN mkdir --parents "${ANDROID_SDK_HOME}/.android/" \ && echo '### User Sources for Android SDK Manager' \ > "${ANDROID_SDK_HOME}/.android/repositories.cfg" -# accept Android licenses (JDK necessary!) +# Download and accept Android licenses (JDK necessary!) RUN ${RETRY} apt -y install -qq --no-install-recommends openjdk-8-jdk \ && apt -y autoremove RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" > /dev/null +RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-27" > /dev/null -# download platforms, API, build tools -RUN "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-19" && \ - "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-27" && \ - "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" && \ - chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager" +# Set avdmanager permissions (executable) +RUN chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager" ENV USER="user" @@ -124,8 +122,6 @@ RUN usermod -append --groups sudo ${USER} RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -RUN pip install --upgrade cython==0.28.6 - WORKDIR ${WORK_DIR} COPY --chown=user:user . ${WORK_DIR} RUN chown --recursive ${USER} ${ANDROID_SDK_HOME} @@ -134,4 +130,5 @@ USER ${USER} # install python-for-android from current branch RUN virtualenv --python=python venv \ && . venv/bin/activate \ + && pip install --upgrade cython==0.28.6 \ && pip install -e . diff --git a/Dockerfile.py3 b/Dockerfile.py3 index 6a8286e9fc..5a3b40a878 100644 --- a/Dockerfile.py3 +++ b/Dockerfile.py3 @@ -58,8 +58,8 @@ RUN ${RETRY} curl --location --progress-bar --insecure \ ENV ANDROID_SDK_HOME="${ANDROID_HOME}/android-sdk" # get the latest version from https://developer.android.com/studio/index.html -ENV ANDROID_SDK_TOOLS_VERSION="3859397" -ENV ANDROID_SDK_BUILD_TOOLS_VERSION="26.0.2" +ENV ANDROID_SDK_TOOLS_VERSION="4333796" +ENV ANDROID_SDK_BUILD_TOOLS_VERSION="28.0.2" ENV ANDROID_SDK_TOOLS_ARCHIVE="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip" ENV ANDROID_SDK_TOOLS_DL_URL="https://dl.google.com/android/repository/${ANDROID_SDK_TOOLS_ARCHIVE}" @@ -76,16 +76,14 @@ RUN mkdir --parents "${ANDROID_SDK_HOME}/.android/" \ && echo '### User Sources for Android SDK Manager' \ > "${ANDROID_SDK_HOME}/.android/repositories.cfg" -# accept Android licenses (JDK necessary!) +# Download and accept Android licenses (JDK necessary!) RUN ${RETRY} apt -y install -qq --no-install-recommends openjdk-8-jdk \ && apt -y autoremove RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" > /dev/null +RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-27" > /dev/null -# download platforms, API, build tools -RUN "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-19" && \ - "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-27" && \ - "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" && \ - chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager" +# Set avdmanager permissions (executable) +RUN chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager" ENV USER="user" @@ -96,7 +94,7 @@ ENV WORK_DIR="${HOME_DIR}" \ # install system dependencies RUN ${RETRY} apt -y install -qq --no-install-recommends \ python3 virtualenv python3-pip python3-venv \ - wget lbzip2 patch sudo \ + wget lbzip2 patch sudo python python-pip \ && apt -y autoremove # build dependencies @@ -124,8 +122,8 @@ RUN useradd --create-home --shell /bin/bash ${USER} RUN usermod -append --groups sudo ${USER} RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers - -RUN pip3 install --upgrade cython==0.28.6 +# install cython for python 2 (for python 3 it's inside the venv) +RUN pip2 install --upgrade Cython==0.28.6 WORKDIR ${WORK_DIR} COPY --chown=user:user . ${WORK_DIR} @@ -135,4 +133,5 @@ USER ${USER} # install python-for-android from current branch RUN virtualenv --python=python3 venv \ && . venv/bin/activate \ + && pip3 install --upgrade Cython==0.28.6 \ && pip3 install -e . diff --git a/ci/constants.py b/ci/constants.py index 27e9d31a1e..c5ab61099d 100644 --- a/ci/constants.py +++ b/ci/constants.py @@ -3,7 +3,6 @@ class TargetPython(Enum): python2 = 0 - python3crystax = 1 python3 = 2 @@ -22,7 +21,6 @@ class TargetPython(Enum): 'ffpyplayer', 'flask', 'groestlcoin_hash', - 'hostpython3crystax', # https://github.com/kivy/python-for-android/issues/1354 'kiwisolver', 'libmysqlclient', @@ -88,5 +86,5 @@ class TargetPython(Enum): # recipes that were already built will be skipped CORE_RECIPES = set([ 'pyjnius', 'kivy', 'openssl', 'requests', 'sqlite3', 'setuptools', - 'numpy', 'android', 'python2', 'python3', + 'numpy', 'android', 'hostpython2', 'hostpython3', 'python2', 'python3', ]) diff --git a/ci/rebuild_updated_recipes.py b/ci/rebuild_updated_recipes.py index 32b0fda0f6..54f62ac768 100755 --- a/ci/rebuild_updated_recipes.py +++ b/ci/rebuild_updated_recipes.py @@ -28,6 +28,7 @@ from pythonforandroid.graph import get_recipe_order_and_bootstrap from pythonforandroid.toolchain import current_directory from pythonforandroid.util import BuildInterruptingException +from pythonforandroid.recipe import Recipe from ci.constants import TargetPython, CORE_RECIPES, BROKEN_RECIPES @@ -66,7 +67,7 @@ def build(target_python, requirements): # iterates to stream the output for line in sh.python( testapp, 'apk', '--sdk-dir', android_sdk_home, - '--ndk-dir', android_ndk_home, '--bootstrap', 'sdl2', '--requirements', + '--ndk-dir', android_ndk_home, '--requirements', requirements, _err_to_out=True, _iter=True): print(line) @@ -78,6 +79,18 @@ def main(): recipes -= CORE_RECIPES logger.info('recipes to build: {}'.format(recipes)) context = Context() + + # removing the deleted recipes for the given target (if any) + for recipe_name in recipes.copy(): + try: + Recipe.get_recipe(recipe_name, context) + except ValueError: + # recipe doesn't exist, so probably we remove it + recipes.remove(recipe_name) + logger.warning( + 'removed {} from recipes because deleted'.format(recipe_name) + ) + # forces the default target recipes_and_target = recipes | set([target_python.name]) try: diff --git a/doc/source/apis.rst b/doc/source/apis.rst index 7c3c307f4e..beae347625 100644 --- a/doc/source/apis.rst +++ b/doc/source/apis.rst @@ -5,6 +5,60 @@ Working on Android This page gives details on accessing Android APIs and managing other interactions on Android. +Storage paths +------------- + +If you want to store and retrieve data, you shouldn't just save to +the current directory, and not hardcode `/sdcard/` or some other +path either - it might differ per device. + +Instead, the `android` module which you can add to your `--requirements` +allows you to query the most commonly required paths:: + + from android.storage import app_storage_path + settings_path = app_storage_path() + + from android.storage import primary_external_storage_path + primary_ext_storage = primary_external_storage_path() + + from android.storage import secondary_external_storage_path + secondary_ext_storage = secondary_external_storage_path() + +`app_storage_path()` gives you Android's so-called "internal storage" +which is specific to your app and cannot seen by others or the user. +It compares best to the AppData directory on Windows. + +`primary_external_storage_path()` returns Android's so-called +"primary external storage", often found at `/sdcard/` and potentially +accessible to any other app. +It compares best to the Documents directory on Windows. +Requires `Permission.WRITE_EXTERNAL_STORAGE` to read and write to. + +`secondary_external_storage_path()` returns Android's so-called +"secondary external storage", often found at `/storage/External_SD/`. +It compares best to an external disk plugged to a Desktop PC, and can +after a device restart become inaccessible if removed. +Requires `Permission.WRITE_EXTERNAL_STORAGE` to read and write to. + +.. warning:: + Even if `secondary_external_storage_path` returns a path + the external sd card may still not be present. + Only non-empty contents or a successful write indicate that it is. + +Read more on all the different storage types and what to use them for +in the Android documentation: + +https://developer.android.com/training/data-storage/files + +A note on permissions +~~~~~~~~~~~~~~~~~~~~~ + +Only the internal storage is always accessible with no additional +permissions. For both primary and secondary external storage, you need +to obtain `Permission.WRITE_EXTERNAL_STORAGE` **and the user may deny it.** +Also, if you get it, both forms of external storage may only allow +your app to write to the common pre-existing folders like "Music", +"Documents", and so on. (see the Android Docs linked above for details) Runtime permissions ------------------- diff --git a/doc/source/buildoptions.rst b/doc/source/buildoptions.rst index f2d7b26f64..5d80020728 100644 --- a/doc/source/buildoptions.rst +++ b/doc/source/buildoptions.rst @@ -33,25 +33,14 @@ e.g. ``--requirements=python3``. CrystaX python3 -############### +~~~~~~~~~~~~~~~ -.. warning:: python-for-android originally supported Python 3 using the CrystaX - NDK. This support is now being phased out as CrystaX is no longer - actively developed. +python-for-android no longer supports building for Python 3 using the CrystaX +NDK. Instead, use the python3 recipe, which can be built using the normal +Google NDK. -.. note:: You must manually download the `CrystaX NDK - `__ and tell - python-for-android to use it with ``--ndk-dir /path/to/NDK``. - -Select this by adding the ``python3crystax`` recipe to your -requirements, e.g. ``--requirements=python3crystax``. - -This uses the prebuilt Python from the `CrystaX NDK -`__, a drop-in replacement for -Google's official NDK which includes many improvements. You -*must* use the CrystaX NDK 10.3.0 or higher when building with -python3. You can get it `here -`__. +.. note:: The last python-for-android version supporting CrystaX was `0.7.0. + `__ .. _bootstrap_build_options: diff --git a/doc/source/contribute.rst b/doc/source/contribute.rst index 2fde4daf17..1de0883c4c 100644 --- a/doc/source/contribute.rst +++ b/doc/source/contribute.rst @@ -79,3 +79,154 @@ Release checklist - [ ] `armeabi-v7a` - [ ] `arm64-v8a` - [ ] Check that the version number is correct + + + +How python-for-android uses `pip` +--------------------------------- + +*Last update: July 2019* + +This section is meant to provide a quick summary how +p4a (=python-for-android) uses pip and python packages in +its build process. +**It is written for a python +packagers point of view, not for regular end users or +contributors,** to assist with making pip developers and +other packaging experts aware of p4a's packaging needs. + +Please note this section just attempts to neutrally list the +current mechanisms, so some of this isn't necessarily meant +to stay but just how things work inside p4a in +this very moment. + + +Basic concepts +~~~~~~~~~~~~~~ + +*(This part repeats other parts of the docs, for the sake of +making this a more independent read)* + +p4a builds & packages a python application for use on Android. +It does this by providing a Java wrapper, and for graphical applications +an SDL2-based wrapper which can be used with the kivy UI toolkit if +desired (or alternatively just plain PySDL2). Any such python application +will of course have further library dependencies to do its work. + +p4a supports two types of package dependencies for a project: + +**Recipe:** install script in custom p4a format. Can either install +C/C++ or other things that cannot be pulled in via pip, or things +that can be installed via pip but break on android by default. +These are maintained primarily inside the p4a source tree by p4a +contributors and interested folks. + +**Python package:** any random pip python package can be directly +installed if it doesn't need adjustments to work for Android. + +p4a will map any dependency to an internal recipe if present, and +otherwise use pip to obtain it regularly from whatever external source. + + +Install process regarding packages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The install/build process of a p4a project, as triggered by the +`p4a apk` command, roughly works as follows in regards to python +packages: + +1. The user has specified a project folder to install. This is either + just a folder with python scripts and a `main.py`, or it may + also have a `pyproject.toml` for a more standardized install. + +2. Dependencies are collected: they can be either specified via + ``--requirements`` as a list of names or pip-style URLs, or p4a + can optionally scan them from a project folder via the + pep517 library (if there is a `pyproject.toml` or `setup.py`). + +3. The collected dependencies are mapped to p4a's recipes if any are + available for them, otherwise they're kept around as external + regular package references. + +4. All the dependencies mapped to recipes are built via p4a's internal + mechanisms to build these recipes. (This may or may not indirectly + use pip, depending on whether the recipe wraps a python package + or not and uses pip to install or not.) + +5. **If the user has specified to install the project in standardized + ways,** then the `setup.py`/whatever build system + of the project will be run. This happens with cross compilation set up + (`CC`/`CFLAGS`/... set to use the + proper toolchain) and a custom site-packages location. + The actual comand is a simple `pip install .` in the project folder + with some extra options: e.g. all dependencies that were already + installed by recipes will be pinned with a `-c` constraints file + to make sure pip won't install them, and build isolation will be + disabled via ``--no-build-isolation`` so pip doesn't reinstall + recipe-packages on its own. + + **If the user has not specified to use standardized build approaches**, + p4a will simply install all the remaining dependencies that weren't + mapped to recipes directly and just plain copy in the user project + without installing. Any `setup.py` or `pyproject.toml` of the user + project will then be ignored in this step. + +6. Google's gradle is invoked to package it all up into an `.apk`. + + +Overall process / package relevant notes for p4a +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Here are some common things worth knowing about python-for-android's +dealing with python packages: + +- Packages will work fine without a recipe if they would also build + on Linux ARM, don't use any API not available in the NDK if they + use native code, and don't use any weird compiler flags the toolchain + doesn't like if they use native code. The package also needs to + work with cross compilation. + +- There is currently no easy way for a package to know it is being + cross-compiled (at least that we know of) other than examining the + `CC` compiler that was set, or that it is being cross-compiled for + Android specifically. If that breaks a package it currently needs + to be worked around with a recipe. + +- If a package does **not** work, p4a developers will often create a + recipe instead of getting upstream to fix it because p4a simply + is too niche. + +- Most packages without native code will just work out of the box. + Many with native code tend not to, especially if complex, e.g. numpy. + +- Anything mapped to a p4a recipe cannot be just reinstalled by pip, + specifically also not inside build isolation as a dependency. + (It *may* work if the patches of the recipe are just relevant + to fix runtime issues.) + Therefore as of now, the best way to deal with this limitation seems + to be to keep build isolation always off. + + +Ideas for the future regarding packaging +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- We in overall prefer to use the recipe mechanism less if we can. + In overall the recipes are just a collection of workarounds. + It may look quite hacky from the outside, since p4a + version pins recipe-wrapped packages usually to make the patches reliably + apply. This creates work for the recipes to be kept up-to-date, and + obviously this approach doesn't scale too well. However, it has ended + up as a quite practical interims solution until better ways are found. + +- Obviously, it would be nice if packages could know they are being + cross-compiled, and for Android specifically. We aren't currently aware + of a good mechanism for that. + +- If pip could actually run the recipes (instead of p4a wrapping pip and + doing so) then this might even allow build isolation to work - but + this might be too complex to get working. It might be more practical + to just gradually reduce the reliance on recipes instead and make + more packages work out of the box. This has been done e.g. with + improvements to the cross-compile environment being set up automatically, + and we're open for any ideas on how to improve this. + diff --git a/doc/source/docker.rst b/doc/source/docker.rst index 3ff9267f73..623e0e6883 100644 --- a/doc/source/docker.rst +++ b/doc/source/docker.rst @@ -17,12 +17,11 @@ already have Docker preinstalled and set up. .. warning:: This approach is highly space unfriendly! The more layers (``commit``) or even Docker images (``build``) you create the more space it'll consume. - Within the Docker image there is Android + Crystax SDK and NDK + various - dependencies. Within the custom diff made by building the distribution - there is another big chunk of space eaten. The very basic stuff such as - a distribution with: CPython 3, setuptools, Python for Android ``android`` - module, SDL2 (+ deps), PyJNIus and Kivy takes almost 13 GB. Check your free - space first! + Within the Docker image there is Android SDK and NDK + various dependencies. + Within the custom diff made by building the distribution there is another + big chunk of space eaten. The very basic stuff such as a distribution with: + CPython 3, setuptools, Python for Android ``android`` module, SDL2 (+ deps), + PyJNIus and Kivy takes almost 2 GB. Check your free space first! 1. Clone the repository:: diff --git a/doc/source/index.rst b/doc/source/index.rst index 16d6162dc3..84a02b962e 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -38,6 +38,7 @@ Contents troubleshooting docker contribute + testing_pull_requests Indices and tables diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index f92b475542..1f50579dde 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -131,9 +131,9 @@ API/NDK API level 21**: Second, install the build-tools. You can use ``$SDK_DIR/tools/bin/sdkmanager --list`` to see all the -possibilities, but 26.0.2 is the latest version at the time of writing:: +possibilities, but 28.0.2 is the latest version at the time of writing:: - $SDK_DIR/tools/bin/sdkmanager "build-tools;26.0.2" + $SDK_DIR/tools/bin/sdkmanager "build-tools;28.0.2" Configure p4a to use your SDK/NDK ````````````````````````````````` diff --git a/doc/source/testing_pull_requests.rst b/doc/source/testing_pull_requests.rst new file mode 100644 index 0000000000..603828db77 --- /dev/null +++ b/doc/source/testing_pull_requests.rst @@ -0,0 +1,226 @@ +Testing an python-for-android pull request +========================================== + +In order to test a pull request, we recommend to consider the following points: + + #. of course, check if the overall thing makes sense + #. is the CI passing? if not what specifically fails + #. is it working locally at compile time? + #. is it working on device at runtime? + +This document will focus on the third point: +`is it working locally at compile time?` so we will give some hints about how +to proceed in order to create a local copy of the pull requests and build an +apk. We expect that the contributors has enough criteria/knowledge to perform +the other steps mentioned, so let's begin... + +To create an apk from a python-for-android pull request we contemplate three +possible scenarios: + + - using python-for-android commands directly from the pull request files + that we want to test, without installing it (the recommended way for most + of the test cases) + - installing python-for-android using the github's branch of the pull request + - using buildozer and a custom app + +We will explain the first two methods using one of the distributed +python-for-android test apps and we assume that you already have the +python-for-android dependencies installed. For the `buildozer` method we also +expect that you already have a a properly working app to test and a working +installation/configuration of `buildozer`. There is one step that it's shared +with all the testing methods that we propose in here...we named it +`Common steps`. + + +Common steps +^^^^^^^^^^^^ +The first step to do it's to get a copy of the pull request, we can do it of +several ways, and that it will depend of the circumstances but all the methods +presented here will do the job, so... + +Fetch the pull request by number +-------------------------------- +For the example, we will use `1901` for the example) and the pull request +branch that we will use is `feature-fix-numpy`, then you will use a variation +of the following git command: +`git fetch origin pull/<#>/head:`, eg.:: + + .. codeblock:: bash + + git fetch upstream pull/1901/head:feature-fix-numpy + +.. note:: Notice that we fetch from `upstream`, since that is the original + project, where the pull request is supposed to be + +.. tip:: The amount of work of some users maybe worth it to add his remote + to your fork's git configuration, to do so with the imaginary + github user `Obi-Wan Kenobi` which nickname is `obiwankenobi`, you + will do:: + + .. codeblock:: bash + + git remote add obiwankenobi https://github.com/obiwankenobi/python-for-android.git + + And to fetch the pull request branch that we put as example, you + would do:: + + .. codeblock:: bash + + git fetch obiwankenobi + git checkout obiwankenobi/feature-fix-numpy + + +Clone the pull request branch from the user's fork +-------------------------------------------------- +Sometimes you may prefer to use directly the fork of the user, so you will get +the nickname of the user who created the pull request, let's take the same +imaginary user than before `obiwankenobi`:: + + .. codeblock:: bash + + git clone -b feature-fix-numpy \ + --single-branch \ + https://github.com/obiwankenobi/python-for-android.git \ + p4a-feature-fix-numpy + +Here's the above command explained line by line: + +- `git clone -b feature-fix-numpy`: we tell git that we want to clone the + branch named `feature-fix-numpy` +- `--single-branch`: we tell git that we only want that branch +- `https://github.com/obiwankenobi/python-for-android.git`: noticed the + nickname of the user that created the pull request: `obiwankenobi` in the + middle of the line? that should be changed as needed for each pull + request that you want to test +- `p4a-feature-fix-numpy`: the name of the cloned repository, so we can + have multiple clones of different prs in the same folder + +.. note:: You can view the author/branch information looking at the + subtitle of the pull request, near the pull request status (expected + an `open` status) + +Using python-for-android commands directly from the pull request files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Enter inside the directory of the cloned repository in the above + step and run p4a command with proper args, eg:: + + .. codeblock:: bash + + cd p4a-feature-fix-numpy + PYTHONPATH=. python3 -m pythonforandroid.toolchain apk \ + --private=testapps/testapp_sqlite_openssl \ + --dist-name=dist_test_app_python3_libs \ + --package=org.kivy \ + --name=test_app_python3_sqlite_openssl \ + --version=0.1 \ + --requirements=requests,peewee,sdl2,pyjnius,kivy,python3 \ + --ndk-dir=/media/DEVEL/Android/android-ndk-r20 \ + --sdk-dir=/media/DEVEL/Android/android-sdk-linux \ + --android-api=27 \ + --arch=arm64-v8a \ + --permission=INTERNET \ + --debug + +Things that you should know: + + + - The example above will build an testapp we will make use of the files of + the testapp named `testapp_sqlite_openssl.py` but we don't use the setup + file to build it so we must tell python-for-android what we want via + arguments + - be sure to at least edit the following arguments when running the above + command, since the default set in there it's unlikely that match your + installation: + + - `--ndk-dir`: An absolute path to your android's NDK dir + - `--sdk-dir`: An absolute path to your android's SDK dir + - `--debug`: this one enables the debug mode of python-for-android, + which will show all log messages of the build. You can omit this + one but it's worth it to be mentioned, since this it's useful to us + when trying to find the source of the problem when things goes + wrong + - The apk generated by the above command should be located at the root of + of the cloned repository, were you run the command to build the apk + - The testapps distributed with python-for-android are located at + `testapps` folder under the main folder project + - All the builds of python-for-android are located at + `~/.local/share/python-for-android` + - You should have a downloaded copy of the android's NDK and SDK + +Installing python-for-android using the github's branch of the pull request +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Enter inside the directory of the cloned repository mentioned in + `Common steps` and install it via pip, eg.:: + + .. codeblock:: bash + + cd p4a-feature-fix-numpy + pip3 install . --upgrade --user + +- Now, go inside the `testapps` directory (we assume that you still are inside + the cloned repository):: + + .. codeblock:: bash + + cd testapps + +- Run the build of the apk via the freshly installed copy of python-for-android + by running a similar command than below:: + + .. code-block:: bash + + python3 setup_testapp_python3_sqlite_openssl.py apk \ + --ndk-dir=/media/DEVEL/Android/android-ndk-r20 \ + --sdk-dir=/media/DEVEL/Android/android-sdk-linux \ + --android-api=27 \ + --arch=arm64-v8a \ + --debug + + +Things that you should know: + + - In the example above, we override some variables that are set in + `setup_testapp_python3_sqlite_openssl.py`, you could also override them + by editing this file + - be sure to at least edit the following arguments when running the above + command, since the default set in there it's unlikely that match your + installation: + + - `--ndk-dir`: An absolute path to your android's NDK dir + - `--sdk-dir`: An absolute path to your android's SDK dir + +.. tip:: if you don't want to mess up with the system's python, you could do + the same steps but inside a virtualenv + +.. warning:: Once you finish the pull request tests remember to go back to the + master or develop versions of python-for-android, since you just + installed the python-for-android files of the `pull request` + +Using buildozer with a custom app +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Edit your `buildozer.spec` file. You should search for the key + `p4a.source_dir` and set the right value so in the example posted in + `Common steps` it would look like this:: + + p4a.source_dir = /home/user/p4a_pull_requests/p4a-feature-fix-numpy + +- Run you buildozer command as usual, eg.:: + + buildozer android debug p4a --dist-name=dist-test-feature-fix-numpy + +.. note:: this method has the advantage, can be run without installing the + pull request version of python-for-android nor the android's + dependencies but has one problem...when things goes wrong you must + determine if it's a buildozer issue or a python-for-android one + +.. warning:: Once you finish the pull request tests remember to comment/edit + the `p4a.source_dir` constant that you just edited to test the + pull request + +.. tip:: this method it's useful for developing pull requests since you can + edit `p4a.source_dir` to point to your python-for-android fork and you + can test any branch you want only switching branches with: + `git checkout ` from inside your python-for-android fork \ No newline at end of file diff --git a/pythonforandroid/__init__.py b/pythonforandroid/__init__.py index 5b93b35870..ee4d0cdbb6 100644 --- a/pythonforandroid/__init__.py +++ b/pythonforandroid/__init__.py @@ -1 +1 @@ -__version__ = '2019.07.08' +__version__ = '2019.08.09' diff --git a/pythonforandroid/archs.py b/pythonforandroid/archs.py index a27067ab1a..42f143ed21 100644 --- a/pythonforandroid/archs.py +++ b/pythonforandroid/archs.py @@ -92,9 +92,6 @@ def get_env(self, with_flags_in_cc=True, clang=False): env["LDFLAGS"] += " ".join(['-lm', '-L' + self.ctx.get_libs_dir(self.arch)]) - if self.ctx.ndk == 'crystax': - env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format(self.ctx.ndk_dir, self.arch) - toolchain_prefix = self.ctx.toolchain_prefix toolchain_version = self.ctx.toolchain_version command_prefix = self.command_prefix @@ -154,10 +151,7 @@ def get_env(self, with_flags_in_cc=True, clang=False): env['LD'] = '{}-ld'.format(command_prefix) env['LDSHARED'] = env["CC"] + " -pthread -shared " +\ "-Wl,-O1 -Wl,-Bsymbolic-functions " - if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax: - # For crystax python, we can't use the host python headers: - env["CFLAGS"] += ' -I{}/sources/python/{}/include/python/'.\ - format(self.ctx.ndk_dir, self.ctx.python_recipe.version[0:3]) + env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix) env['MAKE'] = 'make -j5' env['READELF'] = '{}-readelf'.format(command_prefix) @@ -180,9 +174,6 @@ def get_env(self, with_flags_in_cc=True, clang=False): env['ARCH'] = self.arch env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api)) - if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax: - env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version - return env diff --git a/pythonforandroid/bdistapk.py b/pythonforandroid/bdistapk.py index 6c156ebf39..ea1b78ca2f 100644 --- a/pythonforandroid/bdistapk.py +++ b/pythonforandroid/bdistapk.py @@ -75,7 +75,7 @@ def finalize_options(self): def run(self): self.prepare_build_dir() - from pythonforandroid.toolchain import main + from pythonforandroid.entrypoints import main sys.argv[1] = 'apk' main() diff --git a/pythonforandroid/bootstrap.py b/pythonforandroid/bootstrap.py index 3480b3241b..cd11c6e1a9 100755 --- a/pythonforandroid/bootstrap.py +++ b/pythonforandroid/bootstrap.py @@ -1,10 +1,11 @@ +import functools +import glob +import importlib +import os from os.path import (join, dirname, isdir, normpath, splitext, basename) from os import listdir, walk, sep import sh import shlex -import glob -import importlib -import os import shutil from pythonforandroid.logger import (warning, shprint, info, logger, @@ -34,6 +35,35 @@ def copy_files(src_root, dest_root, override=True): os.makedirs(dest_file) +default_recipe_priorities = [ + "webview", "sdl2", "service_only" # last is highest +] +# ^^ NOTE: these are just the default priorities if no special rules +# apply (which you can find in the code below), so basically if no +# known graphical lib or web lib is used - in which case service_only +# is the most reasonable guess. + + +def _cmp_bootstraps_by_priority(a, b): + def rank_bootstrap(bootstrap): + """ Returns a ranking index for each bootstrap, + with higher priority ranked with higher number. """ + if bootstrap.name in default_recipe_priorities: + return default_recipe_priorities.index(bootstrap.name) + 1 + return 0 + + # Rank bootstraps in order: + rank_a = rank_bootstrap(a) + rank_b = rank_bootstrap(b) + if rank_a != rank_b: + return (rank_b - rank_a) + else: + if a.name < b.name: # alphabetic sort for determinism + return -1 + else: + return 1 + + class Bootstrap(object): '''An Android project template, containing recipe stuff for compilation and templated fields for APK info. @@ -50,10 +80,7 @@ class Bootstrap(object): distribution = None # All bootstraps should include Python in some way: - recipe_depends = [ - ("python2", "python3", "python3crystax"), - 'android', - ] + recipe_depends = [("python2", "python3"), 'android'] can_be_chosen_automatically = True '''Determines whether the bootstrap can be chosen as one that @@ -138,36 +165,43 @@ def run_distribute(self): self.distribution.save_info(self.dist_dir) @classmethod - def list_bootstraps(cls): + def all_bootstraps(cls): '''Find all the available bootstraps and return them.''' forbidden_dirs = ('__pycache__', 'common') bootstraps_dir = join(dirname(__file__), 'bootstraps') + result = set() for name in listdir(bootstraps_dir): if name in forbidden_dirs: continue filen = join(bootstraps_dir, name) if isdir(filen): - yield name + result.add(name) + return result @classmethod - def get_bootstrap_from_recipes(cls, recipes, ctx): - '''Returns a bootstrap whose recipe requirements do not conflict with - the given recipes.''' + def get_usable_bootstraps_for_recipes(cls, recipes, ctx): + '''Returns all bootstrap whose recipe requirements do not conflict + with the given recipes, in no particular order.''' info('Trying to find a bootstrap that matches the given recipes.') bootstraps = [cls.get_bootstrap(name, ctx) - for name in cls.list_bootstraps()] - acceptable_bootstraps = [] + for name in cls.all_bootstraps()] + acceptable_bootstraps = set() + + # Find out which bootstraps are acceptable: for bs in bootstraps: if not bs.can_be_chosen_automatically: continue - possible_dependency_lists = expand_dependencies(bs.recipe_depends) + possible_dependency_lists = expand_dependencies(bs.recipe_depends, ctx) for possible_dependencies in possible_dependency_lists: ok = True + # Check if the bootstap's dependencies have an internal conflict: for recipe in possible_dependencies: recipe = Recipe.get_recipe(recipe, ctx) if any([conflict in recipes for conflict in recipe.conflicts]): ok = False break + # Check if bootstrap's dependencies conflict with chosen + # packages: for recipe in recipes: try: recipe = Recipe.get_recipe(recipe, ctx) @@ -180,14 +214,58 @@ def get_bootstrap_from_recipes(cls, recipes, ctx): ok = False break if ok and bs not in acceptable_bootstraps: - acceptable_bootstraps.append(bs) + acceptable_bootstraps.add(bs) + info('Found {} acceptable bootstraps: {}'.format( len(acceptable_bootstraps), [bs.name for bs in acceptable_bootstraps])) - if acceptable_bootstraps: - info('Using the first of these: {}' - .format(acceptable_bootstraps[0].name)) - return acceptable_bootstraps[0] + return acceptable_bootstraps + + @classmethod + def get_bootstrap_from_recipes(cls, recipes, ctx): + '''Picks a single recommended default bootstrap out of + all_usable_bootstraps_from_recipes() for the given reicpes, + and returns it.''' + + known_web_packages = {"flask"} # to pick webview over service_only + recipes_with_deps_lists = expand_dependencies(recipes, ctx) + acceptable_bootstraps = cls.get_usable_bootstraps_for_recipes( + recipes, ctx + ) + + def have_dependency_in_recipes(dep): + for dep_list in recipes_with_deps_lists: + if dep in dep_list: + return True + return False + + # Special rule: return SDL2 bootstrap if there's an sdl2 dep: + if (have_dependency_in_recipes("sdl2") and + "sdl2" in [b.name for b in acceptable_bootstraps] + ): + info('Using sdl2 bootstrap since it is in dependencies') + return cls.get_bootstrap("sdl2", ctx) + + # Special rule: return "webview" if we depend on common web recipe: + for possible_web_dep in known_web_packages: + if have_dependency_in_recipes(possible_web_dep): + # We have a web package dep! + if "webview" in [b.name for b in acceptable_bootstraps]: + info('Using webview bootstrap since common web packages ' + 'were found {}'.format( + known_web_packages.intersection(recipes) + )) + return cls.get_bootstrap("webview", ctx) + + prioritized_acceptable_bootstraps = sorted( + list(acceptable_bootstraps), + key=functools.cmp_to_key(_cmp_bootstraps_by_priority) + ) + + if prioritized_acceptable_bootstraps: + info('Using the highest ranked/first of these: {}' + .format(prioritized_acceptable_bootstraps[0].name)) + return prioritized_acceptable_bootstraps[0] return None @classmethod @@ -264,9 +342,6 @@ def _unpack_aar(self, aar, arch): def strip_libraries(self, arch): info('Stripping libraries') - if self.ctx.python_recipe.from_crystax: - info('Python was loaded from CrystaX, skipping strip') - return env = arch.get_env() tokens = shlex.split(env['STRIP']) strip = sh.Command(tokens[0]) @@ -299,9 +374,31 @@ def fry_eggs(self, sitepackages): shprint(sh.rm, '-rf', d) -def expand_dependencies(recipes): +def expand_dependencies(recipes, ctx): + """ This function expands to lists of all different available + alternative recipe combinations, with the dependencies added in + ONLY for all the not-with-alternative recipes. + (So this is like the deps graph very simplified and incomplete, but + hopefully good enough for most basic bootstrap compatibility checks) + """ + + # Add in all the deps of recipes where there is no alternative: + recipes_with_deps = list(recipes) + for entry in recipes: + if not isinstance(entry, (tuple, list)) or len(entry) == 1: + if isinstance(entry, (tuple, list)): + entry = entry[0] + try: + recipe = Recipe.get_recipe(entry, ctx) + recipes_with_deps += recipe.depends + except ValueError: + # it's a pure python package without a recipe, so we + # don't know the dependencies...skipping for now + pass + + # Split up lists by available alternatives: recipe_lists = [[]] - for recipe in recipes: + for recipe in recipes_with_deps: if isinstance(recipe, (tuple, list)): new_recipe_lists = [] for alternative in recipe: @@ -311,6 +408,6 @@ def expand_dependencies(recipes): new_recipe_lists.append(new_list) recipe_lists = new_recipe_lists else: - for old_list in recipe_lists: - old_list.append(recipe) + for existing_list in recipe_lists: + existing_list.append(recipe) return recipe_lists diff --git a/pythonforandroid/bootstraps/common/build/build.py b/pythonforandroid/bootstraps/common/build/build.py index 2b69082f14..ed5e708892 100644 --- a/pythonforandroid/bootstraps/common/build/build.py +++ b/pythonforandroid/bootstraps/common/build/build.py @@ -332,9 +332,7 @@ def make_package(args): shutil.copyfile(join(args.private, "main.py"), join(main_py_only_dir, "main.py")) tar_dirs.append(main_py_only_dir) - for python_bundle_dir in ('private', - 'crystax_python', - '_python_bundle'): + for python_bundle_dir in ('private', '_python_bundle'): if exists(python_bundle_dir): tar_dirs.append(python_bundle_dir) if get_bootstrap_name() == "webview": @@ -783,14 +781,13 @@ def _read_configuration(): if args.try_system_python_compile: # Hardcoding python2.7 is okay for now, as python3 skips the # compilation anyway - if not exists('crystax_python'): - python_executable = 'python2.7' - try: - subprocess.call([python_executable, '--version']) - except (OSError, subprocess.CalledProcessError): - pass - else: - PYTHON = python_executable + python_executable = 'python2.7' + try: + subprocess.call([python_executable, '--version']) + except (OSError, subprocess.CalledProcessError): + pass + else: + PYTHON = python_executable if args.no_compile_pyo: PYTHON = None diff --git a/pythonforandroid/bootstraps/common/build/jni/application/src/Android.mk b/pythonforandroid/bootstraps/common/build/jni/application/src/Android.mk index 4a442eeb32..fb2b17719d 100644 --- a/pythonforandroid/bootstraps/common/build/jni/application/src/Android.mk +++ b/pythonforandroid/bootstraps/common/build/jni/application/src/Android.mk @@ -21,7 +21,3 @@ LOCAL_LDLIBS := -lGLESv1_CM -lGLESv2 -llog $(EXTRA_LDLIBS) LOCAL_LDFLAGS += -L$(PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS) include $(BUILD_SHARED_LIBRARY) - -ifdef CRYSTAX_PYTHON_VERSION - $(call import-module,python/$(CRYSTAX_PYTHON_VERSION)) -endif diff --git a/pythonforandroid/bootstraps/common/build/jni/application/src/start.c b/pythonforandroid/bootstraps/common/build/jni/application/src/start.c index a88ec74c74..24297accdb 100644 --- a/pythonforandroid/bootstraps/common/build/jni/application/src/start.c +++ b/pythonforandroid/bootstraps/common/build/jni/application/src/start.c @@ -165,26 +165,14 @@ int main(int argc, char *argv[]) { // Set up the python path char paths[256]; - char crystax_python_dir[256]; - snprintf(crystax_python_dir, 256, - "%s/crystax_python", getenv("ANDROID_UNPACK")); char python_bundle_dir[256]; snprintf(python_bundle_dir, 256, "%s/_python_bundle", getenv("ANDROID_UNPACK")); - if (dir_exists(crystax_python_dir) || dir_exists(python_bundle_dir)) { - if (dir_exists(crystax_python_dir)) { - LOGP("crystax_python exists"); - snprintf(paths, 256, - "%s/stdlib.zip:%s/modules", - crystax_python_dir, crystax_python_dir); - } - - if (dir_exists(python_bundle_dir)) { - LOGP("_python_bundle dir exists"); - snprintf(paths, 256, - "%s/stdlib.zip:%s/modules", - python_bundle_dir, python_bundle_dir); - } + if (dir_exists(python_bundle_dir)) { + LOGP("_python_bundle dir exists"); + snprintf(paths, 256, + "%s/stdlib.zip:%s/modules", + python_bundle_dir, python_bundle_dir); LOGP("calculated paths to be..."); LOGP(paths); @@ -196,10 +184,8 @@ int main(int argc, char *argv[]) { LOGP("set wchar paths..."); } else { - // We do not expect to see crystax_python any more, so no point - // reminding the user about it. If it does exist, we'll have - // logged it earlier. - LOGP("_python_bundle does not exist"); + LOGP("_python_bundle does not exist...this not looks good, all python" + " recipes should have this folder, should we expect a crash soon?"); } Py_Initialize(); @@ -234,18 +220,6 @@ int main(int argc, char *argv[]) { PyRun_SimpleString("import sys, posix\n"); char add_site_packages_dir[256]; - if (dir_exists(crystax_python_dir)) { - snprintf(add_site_packages_dir, 256, - "sys.path.append('%s/site-packages')", - crystax_python_dir); - - PyRun_SimpleString("import sys\n" - "sys.argv = ['notaninterpreterreally']\n" - "from os.path import realpath, join, dirname"); - PyRun_SimpleString(add_site_packages_dir); - /* "sys.path.append(join(dirname(realpath(__file__)), 'site-packages'))") */ - PyRun_SimpleString("sys.path = ['.'] + sys.path"); - } if (dir_exists(python_bundle_dir)) { snprintf(add_site_packages_dir, 256, diff --git a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonService.java b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonService.java index 7456c617b3..6d951e8525 100644 --- a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonService.java +++ b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonService.java @@ -35,8 +35,11 @@ public class PythonService extends Service implements Runnable { private String pythonHome; private String pythonPath; private String serviceEntrypoint; + private boolean serviceStartAsForeground; // Argument to pass to Python code, private String pythonServiceArgument; + + public static PythonService mService = null; private Intent startIntent = null; @@ -46,10 +49,6 @@ public void setAutoRestartService(boolean restart) { autoRestartService = restart; } - public boolean canDisplayNotification() { - return true; - } - public int startType() { return START_NOT_STICKY; } @@ -79,18 +78,24 @@ public int onStartCommand(Intent intent, int flags, int startId) { pythonName = extras.getString("pythonName"); pythonHome = extras.getString("pythonHome"); pythonPath = extras.getString("pythonPath"); + serviceStartAsForeground = ( + extras.getString("serviceStartAsForeground") == "true" + ); pythonServiceArgument = extras.getString("pythonServiceArgument"); - pythonThread = new Thread(this); pythonThread.start(); - if (canDisplayNotification()) { + if (serviceStartAsForeground) { doStartForeground(extras); } return startType(); } + protected int getServiceId() { + return 1; + } + protected void doStartForeground(Bundle extras) { String serviceTitle = extras.getString("serviceTitle"); String serviceDescription = extras.getString("serviceDescription"); @@ -116,7 +121,7 @@ protected void doStartForeground(Bundle extras) { // for android 8+ we need to create our own channel // https://stackoverflow.com/questions/47531742/startforeground-fail-after-upgrade-to-android-8-1 String NOTIFICATION_CHANNEL_ID = "org.kivy.p4a"; //TODO: make this configurable - String channelName = "PythonSerice"; //TODO: make this configurable + String channelName = "Background Service"; //TODO: make this configurable NotificationChannel chan = new NotificationChannel(NOTIFICATION_CHANNEL_ID, channelName, NotificationManager.IMPORTANCE_NONE); @@ -132,7 +137,7 @@ protected void doStartForeground(Bundle extras) { builder.setSmallIcon(context.getApplicationInfo().icon); notification = builder.build(); } - startForeground(1, notification); + startForeground(getServiceId(), notification); } @Override diff --git a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java index 1f2673850d..2bb1cf3607 100644 --- a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java +++ b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java @@ -32,7 +32,6 @@ protected static void addLibraryIfExists(ArrayList libsList, String patt protected static ArrayList getLibraries(File libsDir) { ArrayList libsList = new ArrayList(); - addLibraryIfExists(libsList, "crystax", libsDir); addLibraryIfExists(libsList, "sqlite3", libsDir); addLibraryIfExists(libsList, "ffi", libsDir); addLibraryIfExists(libsList, "ssl.*", libsDir); diff --git a/pythonforandroid/bootstraps/common/build/templates/Service.tmpl.java b/pythonforandroid/bootstraps/common/build/templates/Service.tmpl.java index 3ed10c2690..50c08f08a1 100644 --- a/pythonforandroid/bootstraps/common/build/templates/Service.tmpl.java +++ b/pythonforandroid/bootstraps/common/build/templates/Service.tmpl.java @@ -1,15 +1,8 @@ package {{ args.package }}; -import android.os.Build; -import java.lang.reflect.Method; -import java.lang.reflect.InvocationTargetException; import android.content.Intent; import android.content.Context; -import android.app.Notification; -import android.app.PendingIntent; -import android.os.Bundle; import org.kivy.android.PythonService; -import org.kivy.android.PythonActivity; public class Service{{ name|capitalize }} extends PythonService { @@ -20,41 +13,9 @@ public int startType() { } {% endif %} - {% if not foreground %} @Override - public boolean canDisplayNotification() { - return false; - } - {% endif %} - - @Override - protected void doStartForeground(Bundle extras) { - Notification notification; - Context context = getApplicationContext(); - Intent contextIntent = new Intent(context, PythonActivity.class); - PendingIntent pIntent = PendingIntent.getActivity(context, 0, contextIntent, - PendingIntent.FLAG_UPDATE_CURRENT); - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.HONEYCOMB) { - notification = new Notification( - context.getApplicationInfo().icon, "{{ args.name }}", System.currentTimeMillis()); - try { - // prevent using NotificationCompat, this saves 100kb on apk - Method func = notification.getClass().getMethod( - "setLatestEventInfo", Context.class, CharSequence.class, - CharSequence.class, PendingIntent.class); - func.invoke(notification, context, "{{ args.name }}", "{{ name| capitalize }}", pIntent); - } catch (NoSuchMethodException | IllegalAccessException | - IllegalArgumentException | InvocationTargetException e) { - } - } else { - Notification.Builder builder = new Notification.Builder(context); - builder.setContentTitle("{{ args.name }}"); - builder.setContentText("{{ name| capitalize }}"); - builder.setContentIntent(pIntent); - builder.setSmallIcon(context.getApplicationInfo().icon); - notification = builder.build(); - } - startForeground({{ service_id }}, notification); + protected int getServiceId() { + return {{ service_id }}; } static public void start(Context ctx, String pythonServiceArgument) { @@ -62,8 +23,11 @@ static public void start(Context ctx, String pythonServiceArgument) { String argument = ctx.getFilesDir().getAbsolutePath() + "/app"; intent.putExtra("androidPrivate", ctx.getFilesDir().getAbsolutePath()); intent.putExtra("androidArgument", argument); + intent.putExtra("serviceTitle", "{{ args.name }}"); + intent.putExtra("serviceDescription", "{{ name|capitalize }}"); intent.putExtra("serviceEntrypoint", "{{ entrypoint }}"); intent.putExtra("pythonName", "{{ name }}"); + intent.putExtra("serviceStartAsForeground", "{{ foreground|lower }}"); intent.putExtra("pythonHome", argument); intent.putExtra("pythonPath", argument + ":" + argument + "/lib"); intent.putExtra("pythonServiceArgument", pythonServiceArgument); diff --git a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java index 425923433f..33d0855ef3 100644 --- a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java @@ -365,8 +365,32 @@ protected void onActivityResult(int requestCode, int resultCode, Intent intent) } } - public static void start_service(String serviceTitle, String serviceDescription, - String pythonServiceArgument) { + public static void start_service( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument + ) { + _do_start_service( + serviceTitle, serviceDescription, pythonServiceArgument, true + ); + } + + public static void start_service_not_as_foreground( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument + ) { + _do_start_service( + serviceTitle, serviceDescription, pythonServiceArgument, false + ); + } + + public static void _do_start_service( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument, + boolean showForegroundNotification + ) { Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class); String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath(); String filesDirectory = argument; @@ -378,6 +402,9 @@ public static void start_service(String serviceTitle, String serviceDescription, serviceIntent.putExtra("pythonName", "python"); serviceIntent.putExtra("pythonHome", app_root_dir); serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib"); + serviceIntent.putExtra("serviceStartAsForeground", + (showForegroundNotification ? "true" : "false") + ); serviceIntent.putExtra("serviceTitle", serviceTitle); serviceIntent.putExtra("serviceDescription", serviceDescription); serviceIntent.putExtra("pythonServiceArgument", pythonServiceArgument); @@ -581,7 +608,34 @@ public void onWindowFocusChanged(boolean hasFocus) { // call native function (since it's not yet loaded) } considerLoadingScreenRemoval(); - } + } + + /** + * Used by android.permissions p4a module to register a call back after + * requesting runtime permissions + **/ + public interface PermissionsCallback { + void onRequestPermissionsResult(int requestCode, String[] permissions, int[] grantResults); + } + + private PermissionsCallback permissionCallback; + private boolean havePermissionsCallback = false; + + public void addPermissionsCallback(PermissionsCallback callback) { + permissionCallback = callback; + havePermissionsCallback = true; + Log.v(TAG, "addPermissionsCallback(): Added callback for onRequestPermissionsResult"); + } + + @Override + public void onRequestPermissionsResult(int requestCode, String[] permissions, int[] grantResults) { + Log.v(TAG, "onRequestPermissionsResult()"); + if (havePermissionsCallback) { + Log.v(TAG, "onRequestPermissionsResult passed to callback"); + permissionCallback.onRequestPermissionsResult(requestCode, permissions, grantResults); + } + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + } /** * Used by android.permissions p4a module to check a permission @@ -592,9 +646,9 @@ public boolean checkCurrentPermission(String permission) { try { java.lang.reflect.Method methodCheckPermission = - Activity.class.getMethod("checkSelfPermission", java.lang.String.class); + Activity.class.getMethod("checkSelfPermission", java.lang.String.class); Object resultObj = methodCheckPermission.invoke(this, permission); - int result = Integer.parseInt(resultObj.toString()); + int result = Integer.parseInt(resultObj.toString()); if (result == PackageManager.PERMISSION_GRANTED) return true; } catch (IllegalAccessException | NoSuchMethodException | @@ -606,16 +660,20 @@ public boolean checkCurrentPermission(String permission) { /** * Used by android.permissions p4a module to request runtime permissions **/ - public void requestPermissions(String[] permissions) { + public void requestPermissionsWithRequestCode(String[] permissions, int requestCode) { if (android.os.Build.VERSION.SDK_INT < 23) return; try { java.lang.reflect.Method methodRequestPermission = Activity.class.getMethod("requestPermissions", - java.lang.String[].class, int.class); - methodRequestPermission.invoke(this, permissions, 1); + java.lang.String[].class, int.class); + methodRequestPermission.invoke(this, permissions, requestCode); } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { } } + + public void requestPermissions(String[] permissions) { + requestPermissionsWithRequestCode(permissions, 1); + } } diff --git a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonUtil.java b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonUtil.java index c1180addfa..d7eb704f12 100644 --- a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonUtil.java +++ b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonUtil.java @@ -31,7 +31,6 @@ protected static void addLibraryIfExists(ArrayList libsList, String patt protected static ArrayList getLibraries(File libsDir) { ArrayList libsList = new ArrayList(); - addLibraryIfExists(libsList, "crystax", libsDir); addLibraryIfExists(libsList, "sqlite3", libsDir); addLibraryIfExists(libsList, "ffi", libsDir); addLibraryIfExists(libsList, "png16", libsDir); diff --git a/pythonforandroid/bootstraps/service_only/build/jni/application/src/Android.mk b/pythonforandroid/bootstraps/service_only/build/jni/application/src/Android.mk index 0bc42bfb89..dc351a3319 100644 --- a/pythonforandroid/bootstraps/service_only/build/jni/application/src/Android.mk +++ b/pythonforandroid/bootstraps/service_only/build/jni/application/src/Android.mk @@ -16,7 +16,3 @@ LOCAL_LDLIBS := -llog $(EXTRA_LDLIBS) LOCAL_LDFLAGS += -L$(PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS) include $(BUILD_SHARED_LIBRARY) - -ifdef CRYSTAX_PYTHON_VERSION - $(call import-module,python/$(CRYSTAX_PYTHON_VERSION)) -endif diff --git a/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java index 8e45967616..85fc88150d 100644 --- a/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java @@ -380,8 +380,32 @@ protected void onActivityResult(int requestCode, int resultCode, Intent intent) } } - public static void start_service(String serviceTitle, String serviceDescription, - String pythonServiceArgument) { + public static void start_service( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument + ) { + _do_start_service( + serviceTitle, serviceDescription, pythonServiceArgument, true + ); + } + + public static void start_service_not_as_foreground( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument + ) { + _do_start_service( + serviceTitle, serviceDescription, pythonServiceArgument, false + ); + } + + public static void _do_start_service( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument, + boolean showForegroundNotification + ) { Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class); String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath(); String filesDirectory = argument; @@ -393,6 +417,9 @@ public static void start_service(String serviceTitle, String serviceDescription, serviceIntent.putExtra("pythonName", "python"); serviceIntent.putExtra("pythonHome", app_root_dir); serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib"); + serviceIntent.putExtra("serviceStartAsForeground", + (showForegroundNotification ? "true" : "false") + ); serviceIntent.putExtra("serviceTitle", serviceTitle); serviceIntent.putExtra("serviceDescription", serviceDescription); serviceIntent.putExtra("pythonServiceArgument", pythonServiceArgument); diff --git a/pythonforandroid/bootstraps/service_only/build/templates/Service.tmpl.java b/pythonforandroid/bootstraps/service_only/build/templates/Service.tmpl.java index ecbf3fe961..598549d345 100644 --- a/pythonforandroid/bootstraps/service_only/build/templates/Service.tmpl.java +++ b/pythonforandroid/bootstraps/service_only/build/templates/Service.tmpl.java @@ -22,15 +22,10 @@ public int startType() { } {% endif %} - {% if foreground %} - /** - * {@inheritDoc} - */ @Override - public boolean getStartForeground() { - return true; + protected int getServiceId() { + return {{ service_id }}; } - {% endif %} public static void start(Context ctx, String pythonServiceArgument) { String argument = ctx.getFilesDir().getAbsolutePath() + "/app"; @@ -41,6 +36,7 @@ public static void start(Context ctx, String pythonServiceArgument) { intent.putExtra("serviceTitle", "{{ name|capitalize }}"); intent.putExtra("serviceDescription", ""); intent.putExtra("pythonName", "{{ name }}"); + intent.putExtra("serviceStartAsForeground", "{{ foreground|lower }}"); intent.putExtra("pythonHome", argument); intent.putExtra("androidUnpack", argument); intent.putExtra("pythonPath", argument + ":" + argument + "/lib"); diff --git a/pythonforandroid/bootstraps/webview/build/jni/application/src/Android.mk b/pythonforandroid/bootstraps/webview/build/jni/application/src/Android.mk index 20399573c9..5fbc4cd365 100644 --- a/pythonforandroid/bootstraps/webview/build/jni/application/src/Android.mk +++ b/pythonforandroid/bootstraps/webview/build/jni/application/src/Android.mk @@ -18,7 +18,3 @@ LOCAL_LDLIBS := -llog $(EXTRA_LDLIBS) LOCAL_LDFLAGS += -L$(PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS) include $(BUILD_SHARED_LIBRARY) - -ifdef CRYSTAX_PYTHON_VERSION - $(call import-module,python/$(CRYSTAX_PYTHON_VERSION)) -endif diff --git a/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java index effa54cf36..2bd908ce92 100644 --- a/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java @@ -437,8 +437,32 @@ protected void onActivityResult(int requestCode, int resultCode, Intent intent) } } - public static void start_service(String serviceTitle, String serviceDescription, - String pythonServiceArgument) { + public static void start_service( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument + ) { + _do_start_service( + serviceTitle, serviceDescription, pythonServiceArgument, true + ); + } + + public static void start_service_not_as_foreground( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument + ) { + _do_start_service( + serviceTitle, serviceDescription, pythonServiceArgument, false + ); + } + + public static void _do_start_service( + String serviceTitle, + String serviceDescription, + String pythonServiceArgument, + boolean showForegroundNotification + ) { Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class); String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath(); String filesDirectory = argument; @@ -450,6 +474,9 @@ public static void start_service(String serviceTitle, String serviceDescription, serviceIntent.putExtra("pythonName", "python"); serviceIntent.putExtra("pythonHome", app_root_dir); serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib"); + serviceIntent.putExtra("serviceStartAsForeground", + (showForegroundNotification ? "true" : "false") + ); serviceIntent.putExtra("serviceTitle", serviceTitle); serviceIntent.putExtra("serviceDescription", serviceDescription); serviceIntent.putExtra("pythonServiceArgument", pythonServiceArgument); diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index 19893d1619..16cc459b4c 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -27,6 +27,58 @@ RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API) +def get_ndk_platform_dir(ndk_dir, ndk_api, arch): + ndk_platform_dir_exists = True + platform_dir = arch.platform_dir + ndk_platform = join( + ndk_dir, + 'platforms', + 'android-{}'.format(ndk_api), + platform_dir) + if not exists(ndk_platform): + warning("ndk_platform doesn't exist: {}".format(ndk_platform)) + ndk_platform_dir_exists = False + return ndk_platform, ndk_platform_dir_exists + + +def get_toolchain_versions(ndk_dir, arch): + toolchain_versions = [] + toolchain_path_exists = True + toolchain_prefix = arch.toolchain_prefix + toolchain_path = join(ndk_dir, 'toolchains') + if isdir(toolchain_path): + toolchain_contents = glob.glob('{}/{}-*'.format(toolchain_path, + toolchain_prefix)) + toolchain_versions = [split(path)[-1][len(toolchain_prefix) + 1:] + for path in toolchain_contents] + else: + warning('Could not find toolchain subdirectory!') + toolchain_path_exists = False + return toolchain_versions, toolchain_path_exists + + +def get_targets(sdk_dir): + if exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')): + avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin', 'avdmanager')) + targets = avdmanager('list', 'target').stdout.decode('utf-8').split('\n') + elif exists(join(sdk_dir, 'tools', 'android')): + android = sh.Command(join(sdk_dir, 'tools', 'android')) + targets = android('list').stdout.decode('utf-8').split('\n') + else: + raise BuildInterruptingException( + 'Could not find `android` or `sdkmanager` binaries in Android SDK', + instructions='Make sure the path to the Android SDK is correct') + return targets + + +def get_available_apis(sdk_dir): + targets = get_targets(sdk_dir) + apis = [s for s in targets if re.match(r'^ *API level: ', s)] + apis = [re.findall(r'[0-9]+', s) for s in apis] + apis = [int(s[0]) for s in apis if s] + return apis + + class Context(object): '''A build context. If anything will be built, an instance this class will be instantiated and used to hold all the build state.''' @@ -51,7 +103,6 @@ class Context(object): use_setup_py = False ccache = None # whether to use ccache - cython = None # the cython interpreter name ndk_platform = None # the ndk platform directory @@ -238,20 +289,7 @@ def prepare_build_environment(self, self.android_api = android_api check_target_api(android_api, self.archs[0].arch) - - if exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')): - avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin', 'avdmanager')) - targets = avdmanager('list', 'target').stdout.decode('utf-8').split('\n') - elif exists(join(sdk_dir, 'tools', 'android')): - android = sh.Command(join(sdk_dir, 'tools', 'android')) - targets = android('list').stdout.decode('utf-8').split('\n') - else: - raise BuildInterruptingException( - 'Could not find `android` or `sdkmanager` binaries in Android SDK', - instructions='Make sure the path to the Android SDK is correct') - apis = [s for s in targets if re.match(r'^ *API level: ', s)] - apis = [re.findall(r'[0-9]+', s) for s in apis] - apis = [int(s[0]) for s in apis if s] + apis = get_available_apis(self.sdk_dir) info('Available Android APIs are ({})'.format( ', '.join(map(str, apis)))) if android_api in apis: @@ -327,46 +365,28 @@ def prepare_build_environment(self, if not self.ccache: info('ccache is missing, the build will not be optimized in the ' 'future.') - for cython_fn in ("cython", "cython3", "cython2", "cython-2.7"): - cython = sh.which(cython_fn) - if cython: - self.cython = cython - break - else: - raise BuildInterruptingException('No cython binary found.') - if not self.cython: - ok = False - warning("Missing requirement: cython is not installed") + try: + subprocess.check_output([ + "python3", "-m", "cython", "--help", + ]) + except subprocess.CalledProcessError: + warning('Cython for python3 missing. If you are building for ' + ' a python 3 target (which is the default)' + ' then THINGS WILL BREAK.') # This would need to be changed if supporting multiarch APKs arch = self.archs[0] - platform_dir = arch.platform_dir toolchain_prefix = arch.toolchain_prefix - toolchain_version = None - self.ndk_platform = join( - self.ndk_dir, - 'platforms', - 'android-{}'.format(self.ndk_api), - platform_dir) - if not exists(self.ndk_platform): - warning('ndk_platform doesn\'t exist: {}'.format( - self.ndk_platform)) - ok = False + self.ndk_platform, ndk_platform_dir_exists = get_ndk_platform_dir( + self.ndk_dir, self.ndk_api, arch) + ok = ok and ndk_platform_dir_exists py_platform = sys.platform if py_platform in ['linux2', 'linux3']: py_platform = 'linux' - - toolchain_versions = [] - toolchain_path = join(self.ndk_dir, 'toolchains') - if isdir(toolchain_path): - toolchain_contents = glob.glob('{}/{}-*'.format(toolchain_path, - toolchain_prefix)) - toolchain_versions = [split(path)[-1][len(toolchain_prefix) + 1:] - for path in toolchain_contents] - else: - warning('Could not find toolchain subdirectory!') - ok = False + toolchain_versions, toolchain_path_exists = get_toolchain_versions( + self.ndk_dir, arch) + ok = ok and toolchain_path_exists toolchain_versions.sort() toolchain_versions_gcc = [] @@ -561,10 +581,13 @@ def build_recipes(build_order, python_modules, ctx, project_dir, # 4) biglink everything info_main('# Biglinking object files') - if not ctx.python_recipe or not ctx.python_recipe.from_crystax: + if not ctx.python_recipe: biglink(ctx, arch) else: - info('NDK is crystax, skipping biglink (will this work?)') + warning( + "Context's python recipe found, " + "skipping biglink (will this work?)" + ) # 5) postbuild packages info_main('# Postbuilding recipes') diff --git a/pythonforandroid/entrypoints.py b/pythonforandroid/entrypoints.py new file mode 100644 index 0000000000..1ba6a2601f --- /dev/null +++ b/pythonforandroid/entrypoints.py @@ -0,0 +1,20 @@ +from pythonforandroid.recommendations import check_python_version +from pythonforandroid.util import BuildInterruptingException, handle_build_exception + + +def main(): + """ + Main entrypoint for running python-for-android as a script. + """ + + try: + # Check the Python version before importing anything heavier than + # the util functions. This lets us provide a nice message about + # incompatibility rather than having the interpreter crash if it + # reaches unsupported syntax from a newer Python version. + check_python_version() + + from pythonforandroid.toolchain import ToolchainCL + ToolchainCL() + except BuildInterruptingException as exc: + handle_build_exception(exc) diff --git a/pythonforandroid/logger.py b/pythonforandroid/logger.py index 4aba39fcab..77cb9da323 100644 --- a/pythonforandroid/logger.py +++ b/pythonforandroid/logger.py @@ -6,6 +6,8 @@ from math import log10 from collections import defaultdict from colorama import Style as Colo_Style, Fore as Colo_Fore + +# six import left for Python 2 compatibility during initial Python version check import six # This codecs change fixes a bug with log output, but crashes under python3 diff --git a/pythonforandroid/python.py b/pythonforandroid/python.py index 3a214ee714..ab9035e11a 100755 --- a/pythonforandroid/python.py +++ b/pythonforandroid/python.py @@ -49,13 +49,9 @@ class GuestPythonRecipe(TargetPythonRecipe): this limitation. ''' - from_crystax = False - '''True if the python is used from CrystaX, False otherwise (i.e. if - it is built by p4a).''' - configure_args = () '''The configure arguments needed to build the python recipe. Those are - used in method :meth:`build_arch` (if not overwritten like python3crystax's + used in method :meth:`build_arch` (if not overwritten like python3's recipe does). .. note:: This variable should be properly set in subclass. @@ -108,10 +104,6 @@ def __init__(self, *args, **kwargs): super(GuestPythonRecipe, self).__init__(*args, **kwargs) def get_recipe_env(self, arch=None, with_flags_in_cc=True): - if self.from_crystax: - return super(GuestPythonRecipe, self).get_recipe_env( - arch=arch, with_flags_in_cc=with_flags_in_cc) - env = environ.copy() android_host = env['HOSTARCH'] = arch.command_prefix @@ -215,10 +207,6 @@ def add_flags(include_flags, link_dirs, link_libs): def prebuild_arch(self, arch): super(TargetPythonRecipe, self).prebuild_arch(arch) - if self.from_crystax and self.ctx.ndk != 'crystax': - raise BuildInterruptingException( - 'The {} recipe can only be built when using the CrystaX NDK. ' - 'Exiting.'.format(self.name)) self.ctx.python_recipe = self def build_arch(self, arch): diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 9b07a3cf1b..7d2c4ac4e8 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -156,10 +156,10 @@ def report_hook(index, blksize, size): while True: try: urlretrieve(url, target, report_hook) - except OSError as e: + except OSError: attempts += 1 if attempts >= 5: - raise e + raise stdout.write('Download failed retrying in a second...') time.sleep(1) continue @@ -725,14 +725,37 @@ class PythonRecipe(Recipe): This is almost always what you want to do.''' setup_extra_args = [] - '''List of extra arugments to pass to setup.py''' + '''List of extra arguments to pass to setup.py''' + + depends = [('python2', 'python3')] + ''' + .. note:: it's important to keep this depends as a class attribute outside + `__init__` because sometimes we only initialize the class, so the + `__init__` call won't be called and the deps would be missing + (which breaks the dependency graph computation) + + .. warning:: don't forget to call `super().__init__()` in any recipe's + `__init__`, or otherwise it may not be ensured that it depends + on python2 or python3 which can break the dependency graph + ''' def __init__(self, *args, **kwargs): super(PythonRecipe, self).__init__(*args, **kwargs) - depends = self.depends - depends.append(('python2', 'python3', 'python3crystax')) - depends = list(set(depends)) - self.depends = depends + if not any( + [ + d + for d in {'python2', 'python3', ('python2', 'python3')} + if d in self.depends + ] + ): + # We ensure here that the recipe depends on python even it overrode + # `depends`. We only do this if it doesn't already depend on any + # python, since some recipes intentionally don't depend on/work + # with all python variants + depends = self.depends + depends.append(('python2', 'python3')) + depends = list(set(depends)) + self.depends = depends def clean_build(self, arch=None): super(PythonRecipe, self).clean_build(arch=arch) @@ -753,8 +776,6 @@ def real_hostpython_location(self): host_build = Recipe.get_recipe(host_name, self.ctx).get_build_dir() if host_name in ['hostpython2', 'hostpython3']: return join(host_build, 'native-build', 'python') - elif host_name in ['hostpython3crystax']: - return join(host_build, 'hostpython') else: python_recipe = self.ctx.python_recipe return 'python{}'.format(python_recipe.version) @@ -783,27 +804,16 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): env['LANG'] = "en_GB.UTF-8" if not self.call_hostpython_via_targetpython: - # sets python headers/linkages...depending on python's recipe python_name = self.ctx.python_recipe.name - python_version = self.ctx.python_recipe.version - python_short_version = '.'.join(python_version.split('.')[:2]) - if not self.ctx.python_recipe.from_crystax: - env['CFLAGS'] += ' -I{}'.format( - self.ctx.python_recipe.include_root(arch.arch)) - env['LDFLAGS'] += ' -L{} -lpython{}'.format( - self.ctx.python_recipe.link_root(arch.arch), - self.ctx.python_recipe.major_minor_version_string) - if python_name == 'python3': - env['LDFLAGS'] += 'm' - else: - ndk_dir_python = join(self.ctx.ndk_dir, 'sources', - 'python', python_version) - env['CFLAGS'] += ' -I{} '.format( - join(ndk_dir_python, 'include', - 'python')) - env['LDFLAGS'] += ' -L{}'.format( - join(ndk_dir_python, 'libs', arch.arch)) - env['LDFLAGS'] += ' -lpython{}m'.format(python_short_version) + env['CFLAGS'] += ' -I{}'.format( + self.ctx.python_recipe.include_root(arch.arch) + ) + env['LDFLAGS'] += ' -L{} -lpython{}'.format( + self.ctx.python_recipe.link_root(arch.arch), + self.ctx.python_recipe.major_minor_version_string, + ) + if python_name == 'python3': + env['LDFLAGS'] += 'm' hppath = [] hppath.append(join(dirname(self.hostpython_location), 'Lib')) @@ -951,13 +961,6 @@ class CythonRecipe(PythonRecipe): cython_args = [] call_hostpython_via_targetpython = False - def __init__(self, *args, **kwargs): - super(CythonRecipe, self).__init__(*args, **kwargs) - depends = self.depends - depends.append(('python2', 'python3', 'python3crystax')) - depends = list(set(depends)) - self.depends = depends - def build_arch(self, arch): '''Build any cython components, then install the Python module by calling setup.py install with the target Python dir. @@ -1021,9 +1024,11 @@ def cythonize_file(self, env, build_dir, filename): del cyenv['PYTHONPATH'] if 'PYTHONNOUSERSITE' in cyenv: cyenv.pop('PYTHONNOUSERSITE') - cython = 'cython' if self.ctx.python_recipe.from_crystax else self.ctx.cython - cython_command = sh.Command(cython) - shprint(cython_command, filename, *self.cython_args, _env=cyenv) + python_command = sh.Command("python{}".format( + self.ctx.python_recipe.major_minor_version_string.split(".")[0] + )) + shprint(python_command, "-m", "Cython.Build.Cythonize", + filename, *self.cython_args, _env=cyenv) def cythonize_build(self, env, build_dir="."): if not self.cythonize: @@ -1041,9 +1046,6 @@ def get_recipe_env(self, arch, with_flags_in_cc=True): ' -L{} '.format(self.ctx.libs_dir) + ' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'obj', 'local', arch.arch))) - if self.ctx.python_recipe.from_crystax: - env['LDFLAGS'] = (env['LDFLAGS'] + - ' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'libs', arch.arch))) env['LDSHARED'] = env['CC'] + ' -shared' # shprint(sh.whereis, env['LDSHARED'], _env=env) @@ -1059,24 +1061,6 @@ def get_recipe_env(self, arch, with_flags_in_cc=True): env['LIBLINK_PATH'] = liblink_path ensure_dir(liblink_path) - # Add crystax-specific site packages: - if self.ctx.python_recipe.from_crystax: - command = sh.Command('python{}'.format(self.ctx.python_recipe.version)) - site_packages_dirs = command( - '-c', 'import site; print("\\n".join(site.getsitepackages()))') - site_packages_dirs = site_packages_dirs.stdout.decode('utf-8').split('\n') - if 'PYTHONPATH' in env: - env['PYTHONPATH'] = env['PYTHONPATH'] +\ - ':{}'.format(':'.join(site_packages_dirs)) - else: - env['PYTHONPATH'] = ':'.join(site_packages_dirs) - while env['PYTHONPATH'].find("::") > 0: - env['PYTHONPATH'] = env['PYTHONPATH'].replace("::", ":") - if env['PYTHONPATH'].endswith(":"): - env['PYTHONPATH'] = env['PYTHONPATH'][:-1] - if env['PYTHONPATH'].startswith(":"): - env['PYTHONPATH'] = env['PYTHONPATH'][1:] - return env @@ -1084,20 +1068,12 @@ class TargetPythonRecipe(Recipe): '''Class for target python recipes. Sets ctx.python_recipe to point to itself, so as to know later what kind of Python was built or used.''' - from_crystax = False - '''True if the python is used from CrystaX, False otherwise (i.e. if - it is built by p4a).''' - def __init__(self, *args, **kwargs): self._ctx = None super(TargetPythonRecipe, self).__init__(*args, **kwargs) def prebuild_arch(self, arch): super(TargetPythonRecipe, self).prebuild_arch(arch) - if self.from_crystax and self.ctx.ndk != 'crystax': - raise BuildInterruptingException( - 'The {} recipe can only be built when ' - 'using the CrystaX NDK. Exiting.'.format(self.name)) self.ctx.python_recipe = self def include_root(self, arch): diff --git a/pythonforandroid/recipes/android/src/android/_android.pyx b/pythonforandroid/recipes/android/src/android/_android.pyx index aeaaf2310a..bdca2df454 100644 --- a/pythonforandroid/recipes/android/src/android/_android.pyx +++ b/pythonforandroid/recipes/android/src/android/_android.pyx @@ -280,17 +280,29 @@ class AndroidBrowser(object): import webbrowser webbrowser.register('android', AndroidBrowser) -cdef extern void android_start_service(char *, char *, char *) -def start_service(title=None, description=None, arg=None): - cdef char *j_title = NULL - cdef char *j_description = NULL - if title is not None: - j_title = title - if description is not None: - j_description = description - if arg is not None: - j_arg = arg - android_start_service(j_title, j_description, j_arg) + +def start_service(title="Background Service", + description="", arg="", + as_foreground=True): + # Legacy None value support (for old function signature style): + if title is None: + title = "Background Service" + if description is None: + description = "" + if arg is None: + arg = "" + + # Start service: + mActivity = autoclass('org.kivy.android.PythonActivity').mActivity + if as_foreground: + mActivity.start_service( + title, description, arg + ) + else: + mActivity.start_service_not_as_foreground( + title, description, arg + ) + cdef extern void android_stop_service() def stop_service(): diff --git a/pythonforandroid/recipes/android/src/android/_android_jni.c b/pythonforandroid/recipes/android/src/android/_android_jni.c index 9fea723ed8..cf1b1bf500 100644 --- a/pythonforandroid/recipes/android/src/android/_android_jni.c +++ b/pythonforandroid/recipes/android/src/android/_android_jni.c @@ -201,34 +201,6 @@ void android_get_buildinfo() { } } -void android_start_service(char *title, char *description, char *arg) { - static JNIEnv *env = NULL; - static jclass *cls = NULL; - static jmethodID mid = NULL; - - if (env == NULL) { - env = SDL_ANDROID_GetJNIEnv(); - aassert(env); - cls = (*env)->FindClass(env, JNI_NAMESPACE "/PythonActivity"); - aassert(cls); - mid = (*env)->GetStaticMethodID(env, cls, "start_service", - "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V"); - aassert(mid); - } - - jstring j_title = NULL; - jstring j_description = NULL; - jstring j_arg = NULL; - if ( title != 0 ) - j_title = (*env)->NewStringUTF(env, title); - if ( description != 0 ) - j_description = (*env)->NewStringUTF(env, description); - if ( arg != 0 ) - j_arg = (*env)->NewStringUTF(env, arg); - - (*env)->CallStaticVoidMethod(env, cls, mid, j_title, j_description, j_arg); -} - void android_stop_service() { static JNIEnv *env = NULL; static jclass *cls = NULL; diff --git a/pythonforandroid/recipes/android/src/android/permissions.py b/pythonforandroid/recipes/android/src/android/permissions.py index 46dfc04287..963f48454f 100644 --- a/pythonforandroid/recipes/android/src/android/permissions.py +++ b/pythonforandroid/recipes/android/src/android/permissions.py @@ -1,6 +1,7 @@ +import threading try: - from jnius import autoclass + from jnius import autoclass, PythonJavaClass, java_method except ImportError: # To allow importing by build/manifest-creating code without # pyjnius being present: @@ -422,16 +423,166 @@ class Permission: ) -def request_permissions(permissions): - python_activity = autoclass('org.kivy.android.PythonActivity') - python_activity.requestPermissions(permissions) +PERMISSION_GRANTED = 0 +PERMISSION_DENIED = -1 + + +class _onRequestPermissionsCallback(PythonJavaClass): + """Callback class for registering a Python callback from + onRequestPermissionsResult in PythonActivity. + """ + __javainterfaces__ = ['org.kivy.android.PythonActivity$PermissionsCallback'] + __javacontext__ = 'app' + + def __init__(self, func): + self.func = func + super().__init__() + + @java_method('(I[Ljava/lang/String;[I)V') + def onRequestPermissionsResult(self, requestCode, + permissions, grantResults): + self.func(requestCode, permissions, grantResults) + + +class _RequestPermissionsManager: + """Internal class for requesting Android permissions. + + Permissions are requested through the method 'request_permissions' which + accepts a list of permissions and an optional callback. + + Any callback will asynchronously receive arguments from + onRequestPermissionsResult on PythonActivity after requestPermissions is + called. + + The callback supplied must accept two arguments: 'permissions' and + 'grantResults' (as supplied to onPermissionsCallbackResult). + + Note that for SDK_INT < 23, run-time permissions are not required, and so + the callback will be called immediately. + + The attribute '_java_callback' is initially None, but is set when the first + permissions request is made. It is set to an instance of + onRequestPermissionsCallback, which allows the Java callback to be + propagated to the class method 'python_callback'. This is then, in turn, + used to call an application callback if provided to request_permissions. + + The attribute '_callback_id' is incremented with each call to + request_permissions which has a callback (the value '1' is used for any + call which does not pass a callback). This is passed to requestCode in + the Java call, and used to identify (via the _callbacks dictionary) + the matching call. + """ + _SDK_INT = None + _java_callback = None + _callbacks = {1: None} + _callback_id = 1 + # Lock to prevent multiple calls to request_permissions being handled + # simultaneously (as incrementing _callback_id is not atomic) + _lock = threading.Lock() + + @classmethod + def register_callback(cls): + """Register Java callback for requestPermissions.""" + cls._java_callback = _onRequestPermissionsCallback(cls.python_callback) + python_activity = autoclass('org.kivy.android.PythonActivity') + python_activity.addPermissionsCallback(cls._java_callback) + + @classmethod + def request_permissions(cls, permissions, callback=None): + """Requests Android permissions from PythonActivity. + If 'callback' is supplied, the request is made with a new requestCode + and the callback is stored in the _callbacks dict. When a Java callback + with the matching requestCode is received, callback will be called + with arguments of 'permissions' and 'grant_results'. + """ + if not cls._SDK_INT: + # Get the Android build version and store it + VERSION = autoclass('android.os.Build$VERSION') + cls.SDK_INT = VERSION.SDK_INT + if cls.SDK_INT < 23: + # No run-time permissions needed, return immediately. + if callback: + callback(permissions, [True for x in permissions]) + return + # Request permissions + with cls._lock: + if not cls._java_callback: + cls.register_callback() + python_activity = autoclass('org.kivy.android.PythonActivity') + if not callback: + python_activity.requestPermissions(permissions) + else: + cls._callback_id += 1 + python_activity.requestPermissionsWithRequestCode( + permissions, cls._callback_id) + cls._callbacks[cls._callback_id] = callback + @classmethod + def python_callback(cls, requestCode, permissions, grantResults): + """Calls the relevant callback with arguments of 'permissions' + and 'grantResults'.""" + # Convert from Android codes to True/False + grant_results = [x == PERMISSION_GRANTED for x in grantResults] + if cls._callbacks.get(requestCode): + cls._callbacks[requestCode](permissions, grant_results) -def request_permission(permission): - request_permissions([permission]) + +# Public API methods for requesting permissions + +def request_permissions(permissions, callback=None): + """Requests Android permissions. + + Args: + permissions (str): A list of permissions to requests (str) + callback (callable, optional): A function to call when the request + is completed (callable) + + Returns: + None + + Notes: + + Permission strings can be imported from the 'Permission' class in this + module. For example: + + from android import Permission + permissions_list = [Permission.CAMERA, + Permission.WRITE_EXTERNAL_STORAGE] + + See the p4a source file 'permissions.py' for a list of valid permission + strings (pythonforandroid/recipes/android/src/android/permissions.py). + + Any callback supplied must accept two arguments: + permissions (list of str): A list of permission strings + grant_results (list of bool): A list of bools indicating whether the + respective permission was granted. + See Android documentation for onPermissionsCallbackResult for + further information. + + Note that if the request is interupted the callback may contain an empty + list of permissions, without permissions being granted; the App should + check that each permission requested has been granted. + + Also note that when calling request_permission on SDK_INT < 23, the + callback will be returned immediately as requesting permissions is not + required. + """ + _RequestPermissionsManager.request_permissions(permissions, callback) + + +def request_permission(permission, callback=None): + request_permissions([permission], callback) def check_permission(permission): + """Checks if an app holds the passed permission. + + Args: + - permission An Android permission (str) + + Returns: + bool: True if the app holds the permission given, False otherwise. + """ python_activity = autoclass('org.kivy.android.PythonActivity') result = bool(python_activity.checkCurrentPermission( permission + "" diff --git a/pythonforandroid/recipes/android/src/android/storage.py b/pythonforandroid/recipes/android/src/android/storage.py new file mode 100644 index 0000000000..f4d01403bc --- /dev/null +++ b/pythonforandroid/recipes/android/src/android/storage.py @@ -0,0 +1,115 @@ +from jnius import autoclass, cast +import os + + +Environment = autoclass('android.os.Environment') +File = autoclass('java.io.File') + + +def _android_has_is_removable_func(): + VERSION = autoclass('android.os.Build$VERSION') + return (VERSION.SDK_INT >= 24) + + +def _get_sdcard_path(): + """ Internal function to return getExternalStorageDirectory() + path. This is internal because it may either return the internal, + or an external sd card, depending on the device. + Use primary_external_storage_path() + or secondary_external_storage_path() instead which try to + distinguish this properly. + """ + return ( + Environment.getExternalStorageDirectory().getAbsolutePath() + ) + + +def _get_activity(): + """ + Retrieves the activity from `PythonActivity` fallback to `PythonService`. + """ + PythonActivity = autoclass('org.kivy.android.PythonActivity') + activity = PythonActivity.mActivity + if activity is None: + # assume we're running from the background service + PythonService = autoclass('org.kivy.android.PythonService') + activity = PythonService.mService + return activity + + +def app_storage_path(): + """ Locate the built-in device storage used for this app only. + + This storage is APP-SPECIFIC, and not visible to other apps. + It will be wiped when your app is uninstalled. + + Returns directory path to storage. + """ + activity = _get_activity() + currentActivity = cast('android.app.Activity', activity) + context = cast('android.content.ContextWrapper', + currentActivity.getApplicationContext()) + file_p = cast('java.io.File', context.getFilesDir()) + return os.path.normpath(os.path.abspath( + file_p.getAbsolutePath().replace("/", os.path.sep))) + + +def primary_external_storage_path(): + """ Locate the built-in device storage that user can see via file browser. + Often found at: /sdcard/ + + This is storage is SHARED, and visible to other apps and the user. + It will remain untouched when your app is uninstalled. + + Returns directory path to storage. + + WARNING: You need storage permissions to access this storage. + """ + if _android_has_is_removable_func(): + sdpath = _get_sdcard_path() + # Apparently this can both return primary (built-in) or + # secondary (removable) external storage depending on the device, + # therefore check that we got what we wanted: + if not Environment.isExternalStorageRemovable(File(sdpath)): + return sdpath + if "EXTERNAL_STORAGE" in os.environ: + return os.environ["EXTERNAL_STORAGE"] + raise RuntimeError( + "unexpectedly failed to determine " + + "primary external storage path" + ) + + +def secondary_external_storage_path(): + """ Locate the external SD Card storage, which may not be present. + Often found at: /sdcard/External_SD/ + + This storage is SHARED, visible to other apps, and may not be + be available if the user didn't put in an external SD card. + It will remain untouched when your app is uninstalled. + + Returns None if not found, otherwise path to storage. + + WARNING: You need storage permissions to access this storage. + If it is not writable and presents as empty even with + permissions, then the external sd card may not be present. + """ + if _android_has_is_removable_func: + # See if getExternalStorageDirectory() returns secondary ext storage: + sdpath = _get_sdcard_path() + # Apparently this can both return primary (built-in) or + # secondary (removable) external storage depending on the device, + # therefore check that we got what we wanted: + if Environment.isExternalStorageRemovable(File(sdpath)): + if os.path.exists(sdpath): + return sdpath + + # See if we can take a guess based on environment variables: + p = None + if "SECONDARY_STORAGE" in os.environ: + p = os.environ["SECONDARY_STORAGE"] + elif "EXTERNAL_SDCARD_STORAGE" in os.environ: + p = os.environ["EXTERNAL_SDCARD_STORAGE"] + if p is not None and os.path.exists(p): + return p + return None diff --git a/pythonforandroid/recipes/cymunk/__init__.py b/pythonforandroid/recipes/cymunk/__init__.py index 96d4169710..272c18f9e6 100644 --- a/pythonforandroid/recipes/cymunk/__init__.py +++ b/pythonforandroid/recipes/cymunk/__init__.py @@ -6,7 +6,5 @@ class CymunkRecipe(CythonRecipe): url = 'https://github.com/tito/cymunk/archive/{version}.zip' name = 'cymunk' - depends = [('python2', 'python3crystax', 'python3')] - recipe = CymunkRecipe() diff --git a/pythonforandroid/recipes/ffmpeg/__init__.py b/pythonforandroid/recipes/ffmpeg/__init__.py index 85bdb8ad06..4a04f4844f 100644 --- a/pythonforandroid/recipes/ffmpeg/__init__.py +++ b/pythonforandroid/recipes/ffmpeg/__init__.py @@ -98,20 +98,29 @@ def build_arch(self, arch): '--enable-shared', ] + if 'arm64' in arch.arch: + cross_prefix = 'aarch64-linux-android-' + arch_flag = 'aarch64' + else: + cross_prefix = 'arm-linux-androideabi-' + arch_flag = 'arm' + # android: flags += [ '--target-os=android', - '--cross-prefix=arm-linux-androideabi-', - '--arch=arm', + '--cross-prefix={}'.format(cross_prefix), + '--arch={}'.format(arch_flag), '--sysroot=' + self.ctx.ndk_platform, '--enable-neon', '--prefix={}'.format(realpath('.')), ] - cflags += [ - '-mfpu=vfpv3-d16', - '-mfloat-abi=softfp', - '-fPIC', - ] + + if arch_flag == 'arm': + cflags += [ + '-mfpu=vfpv3-d16', + '-mfloat-abi=softfp', + '-fPIC', + ] env['CFLAGS'] += ' ' + ' '.join(cflags) env['LDFLAGS'] += ' ' + ' '.join(ldflags) @@ -121,7 +130,8 @@ def build_arch(self, arch): shprint(sh.make, '-j4', _env=env) shprint(sh.make, 'install', _env=env) # copy libs: - sh.cp('-a', sh.glob('./lib/lib*.so'), self.ctx.get_libs_dir(arch.arch)) + sh.cp('-a', sh.glob('./lib/lib*.so'), + self.ctx.get_libs_dir(arch.arch)) recipe = FFMpegRecipe() diff --git a/pythonforandroid/recipes/flask/__init__.py b/pythonforandroid/recipes/flask/__init__.py index 1a9b685256..05d59eebdf 100644 --- a/pythonforandroid/recipes/flask/__init__.py +++ b/pythonforandroid/recipes/flask/__init__.py @@ -9,7 +9,7 @@ class FlaskRecipe(PythonRecipe): version = '0.10.1' url = 'https://github.com/pallets/flask/archive/{version}.zip' - depends = [('python2', 'python3', 'python3crystax'), 'setuptools'] + depends = ['setuptools'] python_depends = ['jinja2', 'werkzeug', 'markupsafe', 'itsdangerous', 'click'] diff --git a/pythonforandroid/recipes/genericndkbuild/__init__.py b/pythonforandroid/recipes/genericndkbuild/__init__.py index d91f946c88..e6cccb6e8d 100644 --- a/pythonforandroid/recipes/genericndkbuild/__init__.py +++ b/pythonforandroid/recipes/genericndkbuild/__init__.py @@ -7,7 +7,7 @@ class GenericNDKBuildRecipe(BootstrapNDKRecipe): version = None url = None - depends = [('python2', 'python3', 'python3crystax')] + depends = [('python2', 'python3')] conflicts = ['sdl2'] def should_build(self, arch): diff --git a/pythonforandroid/recipes/hostpython2/__init__.py b/pythonforandroid/recipes/hostpython2/__init__.py index 08d45ba564..36eb178d8a 100644 --- a/pythonforandroid/recipes/hostpython2/__init__.py +++ b/pythonforandroid/recipes/hostpython2/__init__.py @@ -12,7 +12,7 @@ class Hostpython2Recipe(HostPythonRecipe): ''' version = '2.7.15' name = 'hostpython2' - conflicts = ['hostpython3', 'hostpython3crystax'] + conflicts = ['hostpython3'] recipe = Hostpython2Recipe() diff --git a/pythonforandroid/recipes/hostpython3/__init__.py b/pythonforandroid/recipes/hostpython3/__init__.py index 8b268bdd4f..a23f0b9fa2 100644 --- a/pythonforandroid/recipes/hostpython3/__init__.py +++ b/pythonforandroid/recipes/hostpython3/__init__.py @@ -11,7 +11,7 @@ class Hostpython3Recipe(HostPythonRecipe): ''' version = '3.7.1' name = 'hostpython3' - conflicts = ['hostpython2', 'hostpython3crystax'] + conflicts = ['hostpython2'] recipe = Hostpython3Recipe() diff --git a/pythonforandroid/recipes/hostpython3crystax/__init__.py b/pythonforandroid/recipes/hostpython3crystax/__init__.py deleted file mode 100644 index 88cee35938..0000000000 --- a/pythonforandroid/recipes/hostpython3crystax/__init__.py +++ /dev/null @@ -1,44 +0,0 @@ -from pythonforandroid.toolchain import Recipe, shprint -from os.path import join -import sh - - -class Hostpython3CrystaXRecipe(Recipe): - version = 'auto' # the version is taken from the python3crystax recipe - name = 'hostpython3crystax' - - conflicts = ['hostpython2'] - - def get_build_container_dir(self, arch=None): - choices = self.check_recipe_choices() - dir_name = '-'.join([self.name] + choices) - return join(self.ctx.build_dir, 'other_builds', dir_name, 'desktop') - - # def prebuild_armeabi(self): - # # Override hostpython Setup? - # shprint(sh.cp, join(self.get_recipe_dir(), 'Setup'), - # join(self.get_build_dir('armeabi'), 'Modules', 'Setup')) - - def get_build_dir(self, arch=None): - return join(self.get_build_container_dir(), self.name) - - def build_arch(self, arch): - """ - Creates expected build and symlinks system Python version. - """ - self.ctx.hostpython = '/usr/bin/false' - # creates the sub buildir (used by other recipes) - # https://github.com/kivy/python-for-android/issues/1154 - sub_build_dir = join(self.get_build_dir(), 'build') - shprint(sh.mkdir, '-p', sub_build_dir) - python3crystax = self.get_recipe('python3crystax', self.ctx) - system_python = sh.which("python" + python3crystax.version) - if system_python is None: - raise OSError( - ('Trying to use python3crystax=={} but this Python version ' - 'is not installed locally.').format(python3crystax.version)) - link_dest = join(self.get_build_dir(), 'hostpython') - shprint(sh.ln, '-sf', system_python, link_dest) - - -recipe = Hostpython3CrystaXRecipe() diff --git a/pythonforandroid/recipes/jedi/__init__.py b/pythonforandroid/recipes/jedi/__init__.py index 6338a52f24..17168e85a3 100644 --- a/pythonforandroid/recipes/jedi/__init__.py +++ b/pythonforandroid/recipes/jedi/__init__.py @@ -5,8 +5,6 @@ class JediRecipe(PythonRecipe): version = 'v0.9.0' url = 'https://github.com/davidhalter/jedi/archive/{version}.tar.gz' - depends = [('python2', 'python3crystax', 'python3')] - patches = ['fix_MergedNamesDict_get.patch'] # This apparently should be fixed in jedi 0.10 (not released to # pypi yet), but it still occurs on Android, I could not reproduce diff --git a/pythonforandroid/recipes/kivy/__init__.py b/pythonforandroid/recipes/kivy/__init__.py index 689d5da646..3106f25ce6 100644 --- a/pythonforandroid/recipes/kivy/__init__.py +++ b/pythonforandroid/recipes/kivy/__init__.py @@ -6,11 +6,11 @@ class KivyRecipe(CythonRecipe): - version = '1.11.0' + version = '1.11.1' url = 'https://github.com/kivy/kivy/archive/{version}.zip' name = 'kivy' - depends = ['sdl2', 'pyjnius'] + depends = ['sdl2', 'pyjnius', 'setuptools'] def cythonize_build(self, env, build_dir='.'): super(KivyRecipe, self).cythonize_build(env, build_dir=build_dir) diff --git a/pythonforandroid/recipes/libx264/__init__.py b/pythonforandroid/recipes/libx264/__init__.py index c139b4ce74..89d48c8410 100644 --- a/pythonforandroid/recipes/libx264/__init__.py +++ b/pythonforandroid/recipes/libx264/__init__.py @@ -14,9 +14,13 @@ def should_build(self, arch): def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): env = self.get_recipe_env(arch) + if 'arm64' in arch.arch: + cross_prefix = 'aarch64-linux-android-' + else: + cross_prefix = 'arm-linux-androideabi-' configure = sh.Command('./configure') shprint(configure, - '--cross-prefix=arm-linux-androideabi-', + '--cross-prefix={}'.format(cross_prefix), '--host=arm-linux', '--disable-asm', '--disable-cli', diff --git a/pythonforandroid/recipes/libzmq/__init__.py b/pythonforandroid/recipes/libzmq/__init__.py index b7b33aa140..7bf6c2b762 100644 --- a/pythonforandroid/recipes/libzmq/__init__.py +++ b/pythonforandroid/recipes/libzmq/__init__.py @@ -35,6 +35,7 @@ def build_arch(self, arch): '--without-documentation', '--prefix={}'.format(prefix), '--with-libsodium=no', + '--disable-libunwind', _env=env) shprint(sh.make, _env=env) shprint(sh.make, 'install', _env=env) @@ -72,8 +73,8 @@ def get_recipe_env(self, arch): env['CXXFLAGS'] += ' -lgnustl_shared' env['LDFLAGS'] += ' -L{}/sources/cxx-stl/gnu-libstdc++/{}/libs/{}'.format( self.ctx.ndk_dir, self.ctx.toolchain_version, arch) - env['CXXFLAGS'] += ' --sysroot={}/platforms/android-{}/arch-arm'.format( - self.ctx.ndk_dir, self.ctx.ndk_api) + env['CXXFLAGS'] += ' --sysroot={}/platforms/android-{}/{}'.format( + self.ctx.ndk_dir, self.ctx.ndk_api, arch.platform_dir) return env diff --git a/pythonforandroid/recipes/numpy/__init__.py b/pythonforandroid/recipes/numpy/__init__.py index 97df43524d..4e47e9d890 100644 --- a/pythonforandroid/recipes/numpy/__init__.py +++ b/pythonforandroid/recipes/numpy/__init__.py @@ -8,7 +8,6 @@ class NumpyRecipe(CompiledComponentsPythonRecipe): version = '1.16.4' url = 'https://pypi.python.org/packages/source/n/numpy/numpy-{version}.zip' site_packages_name = 'numpy' - depends = [('python2', 'python3', 'python3crystax')] patches = [ join('patches', 'add_libm_explicitly_to_build.patch'), @@ -42,10 +41,6 @@ def get_recipe_env(self, arch): py_ver = self.ctx.python_recipe.major_minor_version_string py_inc_dir = self.ctx.python_recipe.include_root(arch.arch) py_lib_dir = self.ctx.python_recipe.link_root(arch.arch) - if self.ctx.ndk == 'crystax': - src_dir = join(self.ctx.ndk_dir, 'sources') - flags += " -I{}".format(join(src_dir, 'crystax', 'include')) - flags += " -L{}".format(join(src_dir, 'crystax', 'libs', arch.arch)) flags += ' -I{}'.format(py_inc_dir) flags += ' -L{} -lpython{}'.format(py_lib_dir, py_ver) if 'python3' in self.ctx.python_recipe.name: diff --git a/pythonforandroid/recipes/openal/__init__.py b/pythonforandroid/recipes/openal/__init__.py index ad93065f4d..cfb62f6148 100644 --- a/pythonforandroid/recipes/openal/__init__.py +++ b/pythonforandroid/recipes/openal/__init__.py @@ -24,9 +24,6 @@ def build_arch(self, arch): '-DCMAKE_TOOLCHAIN_FILE={}'.format('XCompile-Android.txt'), '-DHOST={}'.format(self.ctx.toolchain_prefix) ] - if self.ctx.ndk == 'crystax': - # avoids a segfault in libcrystax when calling lrintf - cmake_args += ['-DHAVE_LRINTF=0'] shprint( sh.cmake, '.', *cmake_args, diff --git a/pythonforandroid/recipes/openssl/__init__.py b/pythonforandroid/recipes/openssl/__init__.py index 38dbaeeb42..d3033a3594 100644 --- a/pythonforandroid/recipes/openssl/__init__.py +++ b/pythonforandroid/recipes/openssl/__init__.py @@ -20,11 +20,6 @@ class OpenSSLRecipe(Recipe): using the methods: :meth:`include_flags`, :meth:`link_dirs_flags` and :meth:`link_libs_flags`. - .. note:: the python2legacy version is too old to support openssl 1.1+, so - we must use version 1.0.x. Also python3crystax is not building - successfully with openssl libs 1.1+ so we use the legacy version as - we do with python2legacy. - .. warning:: This recipe is very sensitive because is used for our core recipes, the python recipes. The used API should match with the one used in our python build, otherwise we will be unable to build the diff --git a/pythonforandroid/recipes/pysha3/__init__.py b/pythonforandroid/recipes/pysha3/__init__.py index 35cfff84a8..c171c3f662 100644 --- a/pythonforandroid/recipes/pysha3/__init__.py +++ b/pythonforandroid/recipes/pysha3/__init__.py @@ -13,16 +13,11 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super(Pysha3Recipe, self).get_recipe_env(arch, with_flags_in_cc) # CFLAGS may only be used to specify C compiler flags, for macro definitions use CPPFLAGS env['CPPFLAGS'] = env['CFLAGS'] - if self.ctx.ndk == 'crystax': - env['CPPFLAGS'] += ' -I{}/sources/python/{}/include/python/'.format( - self.ctx.ndk_dir, self.ctx.python_recipe.version[0:3]) env['CFLAGS'] = '' # LDFLAGS may only be used to specify linker flags, for libraries use LIBS - env['LDFLAGS'] = env['LDFLAGS'].replace('-lm', '').replace('-lcrystax', '') + env['LDFLAGS'] = env['LDFLAGS'].replace('-lm', '') env['LDFLAGS'] += ' -L{}'.format(os.path.join(self.ctx.bootstrap.build_dir, 'libs', arch.arch)) env['LIBS'] = ' -lm' - if self.ctx.ndk == 'crystax': - env['LIBS'] += ' -lcrystax -lpython{}m'.format(self.ctx.python_recipe.version[0:3]) env['LDSHARED'] += env['LIBS'] return env diff --git a/pythonforandroid/recipes/python2/__init__.py b/pythonforandroid/recipes/python2/__init__.py index ba1fa9a671..e99697f215 100644 --- a/pythonforandroid/recipes/python2/__init__.py +++ b/pythonforandroid/recipes/python2/__init__.py @@ -1,7 +1,7 @@ from os.path import join, exists from pythonforandroid.recipe import Recipe from pythonforandroid.python import GuestPythonRecipe -from pythonforandroid.logger import shprint +from pythonforandroid.logger import shprint, warning import sh @@ -21,7 +21,7 @@ class Python2Recipe(GuestPythonRecipe): name = 'python2' depends = ['hostpython2'] - conflicts = ['python3crystax', 'python3'] + conflicts = ['python3'] patches = [ # new 2.7.15 patches @@ -57,6 +57,11 @@ def prebuild_arch(self, arch): self.apply_patch(join('patches', 'enable-openssl.patch'), arch.arch) shprint(sh.touch, patch_mark) + def build_arch(self, arch): + warning('DEPRECATION: Support for the Python 2 recipe will be ' + 'removed in 2020, please upgrade to Python 3.') + super().build_arch(arch) + def set_libs_flags(self, env, arch): env = super(Python2Recipe, self).set_libs_flags(env, arch) if 'libffi' in self.ctx.recipe_build_order: diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index f22dce8e91..963fad635f 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -28,7 +28,7 @@ class Python3Recipe(GuestPythonRecipe): patches = patches + ["patches/remove-fix-cortex-a8.patch"] depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi'] - conflicts = ['python3crystax', 'python2'] + conflicts = ['python2'] configure_args = ( '--host={android_host}', diff --git a/pythonforandroid/recipes/python3crystax/__init__.py b/pythonforandroid/recipes/python3crystax/__init__.py deleted file mode 100644 index 805be0dd12..0000000000 --- a/pythonforandroid/recipes/python3crystax/__init__.py +++ /dev/null @@ -1,102 +0,0 @@ - -from pythonforandroid.recipe import TargetPythonRecipe -from pythonforandroid.toolchain import shprint -from pythonforandroid.logger import info, error -from pythonforandroid.util import ensure_dir, temp_directory -from os.path import exists, join -import sh - -prebuilt_download_locations = { - '3.6': ('https://github.com/inclement/crystax_python_builds/' - 'releases/download/0.1/crystax_python_3.6_armeabi_armeabi-v7a.tar.gz')} - - -class Python3CrystaXRecipe(TargetPythonRecipe): - version = '3.6' - url = '' - name = 'python3crystax' - - depends = ['hostpython3crystax'] - conflicts = ['python3', 'python2'] - - from_crystax = True - - def get_dir_name(self): - name = super(Python3CrystaXRecipe, self).get_dir_name() - name += '-version{}'.format(self.version) - return name - - def build_arch(self, arch): - # We don't have to actually build anything as CrystaX comes - # with the necessary modules. They are included by modifying - # the Android.mk in the jni folder. - - # If the Python version to be used is not prebuilt with the CrystaX - # NDK, we do have to download it. - - crystax_python_dir = join(self.ctx.ndk_dir, 'sources', 'python') - if not exists(join(crystax_python_dir, self.version)): - info(('The NDK does not have a prebuilt Python {}, trying ' - 'to obtain one.').format(self.version)) - - if self.version not in prebuilt_download_locations: - error(('No prebuilt version for Python {} could be found, ' - 'the built cannot continue.')) - exit(1) - - with temp_directory() as td: - self.download_file(prebuilt_download_locations[self.version], - join(td, 'downloaded_python')) - shprint(sh.tar, 'xf', join(td, 'downloaded_python'), - '--directory', crystax_python_dir) - - if not exists(join(crystax_python_dir, self.version)): - error(('Something went wrong, the directory at {} should ' - 'have been created but does not exist.').format( - join(crystax_python_dir, self.version))) - - if not exists(join( - crystax_python_dir, self.version, 'libs', arch.arch)): - error(('The prebuilt Python for version {} does not contain ' - 'binaries for your chosen architecture "{}".').format( - self.version, arch.arch)) - exit(1) - - # TODO: We should have an option to build a new Python. This - # would also allow linking to openssl and sqlite from CrystaX. - - dirn = self.ctx.get_python_install_dir() - ensure_dir(dirn) - - # Instead of using a locally built hostpython, we use the - # user's Python for now. They must have the right version - # available. Using e.g. pyenv makes this easy. - self.ctx.hostpython = 'python{}'.format(self.version) - - def create_python_bundle(self, dirn, arch): - ndk_dir = self.ctx.ndk_dir - py_recipe = self.ctx.python_recipe - python_dir = join(ndk_dir, 'sources', 'python', - py_recipe.version, 'libs', arch.arch) - shprint(sh.cp, '-r', join(python_dir, - 'stdlib.zip'), dirn) - shprint(sh.cp, '-r', join(python_dir, - 'modules'), dirn) - shprint(sh.cp, '-r', self.ctx.get_python_install_dir(), - join(dirn, 'site-packages')) - - info('Renaming .so files to reflect cross-compile') - self.reduce_object_file_names(join(dirn, "site-packages")) - - return join(dirn, 'site-packages') - - def include_root(self, arch_name): - return join(self.ctx.ndk_dir, 'sources', 'python', self.major_minor_version_string, - 'include', 'python') - - def link_root(self, arch_name): - return join(self.ctx.ndk_dir, 'sources', 'python', self.major_minor_version_string, - 'libs', arch_name) - - -recipe = Python3CrystaXRecipe() diff --git a/pythonforandroid/recipes/secp256k1/__init__.py b/pythonforandroid/recipes/secp256k1/__init__.py index 889803100f..338c7b90c3 100644 --- a/pythonforandroid/recipes/secp256k1/__init__.py +++ b/pythonforandroid/recipes/secp256k1/__init__.py @@ -10,9 +10,14 @@ class Secp256k1Recipe(CppCompiledComponentsPythonRecipe): call_hostpython_via_targetpython = False depends = [ - 'openssl', ('hostpython3', 'hostpython2', 'hostpython3crystax'), - ('python2', 'python3', 'python3crystax'), 'setuptools', - 'libffi', 'cffi', 'libsecp256k1'] + 'openssl', + ('hostpython3', 'hostpython2'), + ('python2', 'python3'), + 'setuptools', + 'libffi', + 'cffi', + 'libsecp256k1' + ] patches = [ "cross_compile.patch", "drop_setup_requires.patch", diff --git a/pythonforandroid/recipes/setuptools/__init__.py b/pythonforandroid/recipes/setuptools/__init__.py index 512d61a843..8e248bd874 100644 --- a/pythonforandroid/recipes/setuptools/__init__.py +++ b/pythonforandroid/recipes/setuptools/__init__.py @@ -6,7 +6,6 @@ class SetuptoolsRecipe(PythonRecipe): url = 'https://pypi.python.org/packages/source/s/setuptools/setuptools-{version}.zip' call_hostpython_via_targetpython = False install_in_hostpython = True - depends = [('python2', 'python3', 'python3crystax')] recipe = SetuptoolsRecipe() diff --git a/pythonforandroid/recipes/six/__init__.py b/pythonforandroid/recipes/six/__init__.py index 2e00432280..ca68e189f8 100644 --- a/pythonforandroid/recipes/six/__init__.py +++ b/pythonforandroid/recipes/six/__init__.py @@ -5,7 +5,7 @@ class SixRecipe(PythonRecipe): version = '1.10.0' url = 'https://pypi.python.org/packages/source/s/six/six-{version}.tar.gz' - depends = [('python2', 'python3', 'python3crystax')] + depends = ['setuptools'] recipe = SixRecipe() diff --git a/pythonforandroid/recommendations.py b/pythonforandroid/recommendations.py index fd2fd3a8be..6cf18eceb9 100644 --- a/pythonforandroid/recommendations.py +++ b/pythonforandroid/recommendations.py @@ -1,7 +1,9 @@ """Simple functions for checking dependency versions.""" +import sys from distutils.version import LooseVersion from os.path import join + from pythonforandroid.logger import info, warning from pythonforandroid.util import BuildInterruptingException @@ -9,29 +11,103 @@ MIN_NDK_VERSION = 17 MAX_NDK_VERSION = 17 -RECOMMENDED_NDK_VERSION = '17c' -OLD_NDK_MESSAGE = 'Older NDKs may not be compatible with all p4a features.' +RECOMMENDED_NDK_VERSION = "17c" +NDK_DOWNLOAD_URL = "https://developer.android.com/ndk/downloads/" + +# Important log messages NEW_NDK_MESSAGE = 'Newer NDKs may not be fully supported by p4a.' +UNKNOWN_NDK_MESSAGE = ( + 'Could not determine NDK version, no source.properties in the NDK dir' +) +PARSE_ERROR_NDK_MESSAGE = ( + 'Could not parse $NDK_DIR/source.properties, not checking NDK version' +) +READ_ERROR_NDK_MESSAGE = ( + 'Unable to read the NDK version from the given directory {ndk_dir}' +) +ENSURE_RIGHT_NDK_MESSAGE = ( + 'Make sure your NDK version is greater than {min_supported}. If you get ' + 'build errors, download the recommended NDK {rec_version} from {ndk_url}' +) +NDK_LOWER_THAN_SUPPORTED_MESSAGE = ( + 'The minimum supported NDK version is {min_supported}. ' + 'You can download it from {ndk_url}' +) +UNSUPPORTED_NDK_API_FOR_ARMEABI_MESSAGE = ( + 'Asked to build for armeabi architecture with API ' + '{req_ndk_api}, but API {max_ndk_api} or greater does not support armeabi' +) +CURRENT_NDK_VERSION_MESSAGE = ( + 'Found NDK version {ndk_version}' +) +RECOMMENDED_NDK_VERSION_MESSAGE = ( + 'Maximum recommended NDK version is {recommended_ndk_version}' +) def check_ndk_version(ndk_dir): - # Check the NDK version against what is currently recommended + """ + Check the NDK version against what is currently recommended and raise an + exception of :class:`~pythonforandroid.util.BuildInterruptingException` in + case that the user tries to use an NDK lower than minimum supported, + specified via attribute `MIN_NDK_VERSION`. + + .. versionchanged:: 2019.06.06.1.dev0 + Added the ability to get android's NDK `letter version` and also + rewrote to raise an exception in case that an NDK version lower than + the minimum supported is detected. + """ version = read_ndk_version(ndk_dir) if version is None: - return # if we failed to read the version, just don't worry about it + warning(READ_ERROR_NDK_MESSAGE.format(ndk_dir=ndk_dir)) + warning( + ENSURE_RIGHT_NDK_MESSAGE.format( + min_supported=MIN_NDK_VERSION, + rec_version=RECOMMENDED_NDK_VERSION, + ndk_url=NDK_DOWNLOAD_URL, + ) + ) + return + + # create a dictionary which will describe the relationship of the android's + # NDK minor version with the `human readable` letter version, egs: + # Pkg.Revision = 17.1.4828580 => ndk-17b + # Pkg.Revision = 17.2.4988734 => ndk-17c + # Pkg.Revision = 19.0.5232133 => ndk-19 (No letter) + minor_to_letter = {0: ''} + minor_to_letter.update( + {n + 1: chr(i) for n, i in enumerate(range(ord('b'), ord('b') + 25))} + ) major_version = version.version[0] + letter_version = minor_to_letter[version.version[1]] + string_version = '{major_version}{letter_version}'.format( + major_version=major_version, letter_version=letter_version + ) - info('Found NDK revision {}'.format(version)) + info(CURRENT_NDK_VERSION_MESSAGE.format(ndk_version=string_version)) if major_version < MIN_NDK_VERSION: - warning('Minimum recommended NDK version is {}'.format( - RECOMMENDED_NDK_VERSION)) - warning(OLD_NDK_MESSAGE) + raise BuildInterruptingException( + NDK_LOWER_THAN_SUPPORTED_MESSAGE.format( + min_supported=MIN_NDK_VERSION, ndk_url=NDK_DOWNLOAD_URL + ), + instructions=( + 'Please, go to the android NDK page ({ndk_url}) and download a' + ' supported version.\n*** The currently recommended NDK' + ' version is {rec_version} ***'.format( + ndk_url=NDK_DOWNLOAD_URL, + rec_version=RECOMMENDED_NDK_VERSION, + ) + ), + ) elif major_version > MAX_NDK_VERSION: - warning('Maximum recommended NDK version is {}'.format( - RECOMMENDED_NDK_VERSION)) + warning( + RECOMMENDED_NDK_VERSION_MESSAGE.format( + recommended_ndk_version=RECOMMENDED_NDK_VERSION + ) + ) warning(NEW_NDK_MESSAGE) @@ -41,16 +117,14 @@ def read_ndk_version(ndk_dir): with open(join(ndk_dir, 'source.properties')) as fileh: ndk_data = fileh.read() except IOError: - info('Could not determine NDK version, no source.properties ' - 'in the NDK dir') + info(UNKNOWN_NDK_MESSAGE) return for line in ndk_data.split('\n'): if line.startswith('Pkg.Revision'): break else: - info('Could not parse $NDK_DIR/source.properties, not checking ' - 'NDK version') + info(PARSE_ERROR_NDK_MESSAGE) return # Line should have the form "Pkg.Revision = ..." @@ -79,9 +153,9 @@ def check_target_api(api, arch): if api >= ARMEABI_MAX_TARGET_API and arch == 'armeabi': raise BuildInterruptingException( - 'Asked to build for armeabi architecture with API ' - '{}, but API {} or greater does not support armeabi'.format( - api, ARMEABI_MAX_TARGET_API), + UNSUPPORTED_NDK_API_FOR_ARMEABI_MESSAGE.format( + req_ndk_api=api, max_ndk_api=ARMEABI_MAX_TARGET_API + ), instructions='You probably want to build with --arch=armeabi-v7a instead') if api < MIN_TARGET_API: @@ -92,16 +166,55 @@ def check_target_api(api, arch): MIN_NDK_API = 21 RECOMMENDED_NDK_API = 21 OLD_NDK_API_MESSAGE = ('NDK API less than {} is not supported'.format(MIN_NDK_API)) +TARGET_NDK_API_GREATER_THAN_TARGET_API_MESSAGE = ( + 'Target NDK API is {ndk_api}, ' + 'higher than the target Android API {android_api}.' +) def check_ndk_api(ndk_api, android_api): """Warn if the user's NDK is too high or low.""" if ndk_api > android_api: raise BuildInterruptingException( - 'Target NDK API is {}, higher than the target Android API {}.'.format( - ndk_api, android_api), + TARGET_NDK_API_GREATER_THAN_TARGET_API_MESSAGE.format( + ndk_api=ndk_api, android_api=android_api + ), instructions=('The NDK API is a minimum supported API number and must be lower ' 'than the target Android API')) if ndk_api < MIN_NDK_API: warning(OLD_NDK_API_MESSAGE) + + +MIN_PYTHON_MAJOR_VERSION = 3 +MIN_PYTHON_MINOR_VERSION = 4 +MIN_PYTHON_VERSION = LooseVersion('{major}.{minor}'.format(major=MIN_PYTHON_MAJOR_VERSION, + minor=MIN_PYTHON_MINOR_VERSION)) +PY2_ERROR_TEXT = ( + 'python-for-android no longer supports running under Python 2. Either upgrade to ' + 'Python {min_version} or higher (recommended), or revert to python-for-android 2019.07.08. ' + 'Note that you *can* still target Python 2 on Android by including python2 in your ' + 'requirements.').format( + min_version=MIN_PYTHON_VERSION) + +PY_VERSION_ERROR_TEXT = ( + 'Your Python version {user_major}.{user_minor} is not supported by python-for-android, ' + 'please upgrade to {min_version} or higher.' + ).format( + user_major=sys.version_info.major, + user_minor=sys.version_info.minor, + min_version=MIN_PYTHON_VERSION) + + +def check_python_version(): + # Python 2 special cased because it's a major transition. In the + # future the major or minor versions can increment more quietly. + if sys.version_info.major == 2: + raise BuildInterruptingException(PY2_ERROR_TEXT) + + if ( + sys.version_info.major < MIN_PYTHON_MAJOR_VERSION or + sys.version_info.minor < MIN_PYTHON_MINOR_VERSION + ): + + raise BuildInterruptingException(PY_VERSION_ERROR_TEXT) diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index b61c9ac54a..cb15ca0392 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -12,7 +12,8 @@ from pythonforandroid.pythonpackage import get_dep_names_of_package from pythonforandroid.recommendations import ( RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API) -from pythonforandroid.util import BuildInterruptingException, handle_build_exception +from pythonforandroid.util import BuildInterruptingException +from pythonforandroid.entrypoints import main def check_python_dependencies(): @@ -568,6 +569,7 @@ def add_parser(subparsers, *args, **kwargs): args, unknown = parser.parse_known_args(sys.argv[1:]) args.unknown_args = unknown + if hasattr(args, "private") and args.private is not None: # Pass this value on to the internal bootstrap build.py: args.unknown_args += ["--private", args.private] @@ -739,6 +741,14 @@ def _read_configuration(): sys.argv.append(arg) def recipes(self, args): + """ + Prints recipes basic info, e.g. + .. code-block:: bash + python3 3.7.1 + depends: ['hostpython3', 'sqlite3', 'openssl', 'libffi'] + conflicts: ['python2'] + optional depends: ['sqlite3', 'libffi', 'openssl'] + """ ctx = self.ctx if args.compact: print(" ".join(set(Recipe.list_recipes(ctx)))) @@ -982,7 +992,7 @@ def apk(self, args): gradlew = sh.Command('./gradlew') if exists('/usr/bin/dos2unix'): # .../dists/bdisttest_python3/gradlew - # .../build/bootstrap_builds/sdl2-python3crystax/gradlew + # .../build/bootstrap_builds/sdl2-python3/gradlew # if docker on windows, gradle contains CRLF output = shprint( sh.Command('dos2unix'), gradlew._path.decode('utf8'), @@ -1178,12 +1188,5 @@ def build_status(self, _args): print(recipe_str) -def main(): - try: - ToolchainCL() - except BuildInterruptingException as exc: - handle_build_exception(exc) - - if __name__ == "__main__": main() diff --git a/pythonforandroid/util.py b/pythonforandroid/util.py index ba392049b6..839858cb1e 100644 --- a/pythonforandroid/util.py +++ b/pythonforandroid/util.py @@ -3,18 +3,17 @@ from os import getcwd, chdir, makedirs, walk, uname import sh import shutil -import sys from fnmatch import fnmatch from tempfile import mkdtemp -try: + +# This Python version workaround left for compatibility during initial version check +try: # Python 3 from urllib.request import FancyURLopener -except ImportError: +except ImportError: # Python 2 from urllib import FancyURLopener from pythonforandroid.logger import (logger, Err_Fore, error, info) -IS_PY3 = sys.version_info[0] >= 3 - class WgetDownloader(FancyURLopener): version = ('Wget/1.17.1') diff --git a/setup.py b/setup.py index 64ab2a0d32..52dc745763 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ data_files = [] + # must be a single statement since buildozer is currently parsing it, refs: # https://github.com/kivy/buildozer/issues/722 install_reqs = [ @@ -94,15 +95,15 @@ def recursively_include(results, directory, patterns): install_requires=install_reqs, entry_points={ 'console_scripts': [ - 'python-for-android = pythonforandroid.toolchain:main', - 'p4a = pythonforandroid.toolchain:main', + 'python-for-android = pythonforandroid.entrypoints:main', + 'p4a = pythonforandroid.entrypoints:main', ], 'distutils.commands': [ 'apk = pythonforandroid.bdistapk:BdistAPK', ], }, classifiers = [ - 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Operating System :: Microsoft :: Windows', @@ -111,7 +112,6 @@ def recursively_include(results, directory, patterns): 'Operating System :: MacOS :: MacOS X', 'Operating System :: Android', 'Programming Language :: C', - 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', 'Topic :: Software Development', 'Topic :: Utilities', diff --git a/testapps/setup_keyboard.py b/testapps/setup_keyboard.py index 026847764d..26499a639a 100644 --- a/testapps/setup_keyboard.py +++ b/testapps/setup_keyboard.py @@ -7,7 +7,7 @@ 'blacklist-requirements': 'openssl,sqlite3', 'android-api': 27, 'ndk-api': 21, - 'ndk-dir': '/home/asandy/android/crystax-ndk-10.3.2', + 'ndk-dir': '/home/asandy/android/android-ndk-r17c', 'dist-name': 'bdisttest', 'ndk-version': '10.3.2', 'permission': 'VIBRATE', diff --git a/testapps/setup_testapp_flask.py b/testapps/setup_testapp_flask.py index 3302e8595c..3b2536e579 100644 --- a/testapps/setup_testapp_flask.py +++ b/testapps/setup_testapp_flask.py @@ -7,7 +7,7 @@ 'blacklist-requirements': 'openssl,sqlite3', 'android-api': 27, 'ndk-api': 21, - 'ndk-dir': '/home/asandy/android/crystax-ndk-10.3.2', + 'ndk-dir': '/home/asandy/android/android-ndk-r17c', 'dist-name': 'testapp_flask', 'ndk-version': '10.3.2', 'bootstrap': 'webview', diff --git a/testapps/setup_testapp_python2.py b/testapps/setup_testapp_python2.py index 5aed64a44a..9499c80c73 100644 --- a/testapps/setup_testapp_python2.py +++ b/testapps/setup_testapp_python2.py @@ -5,7 +5,7 @@ options = {'apk': {'requirements': 'sdl2,numpy,pyjnius,kivy,python2', 'android-api': 27, 'ndk-api': 21, - 'ndk-dir': '/home/asandy/android/crystax-ndk-10.3.2', + 'ndk-dir': '/home/asandy/android/android-ndk-r17c', 'dist-name': 'bdisttest_python2', 'ndk-version': '10.3.2', 'permission': 'VIBRATE', diff --git a/testapps/setup_testapp_python2_sqlite_openssl.py b/testapps/setup_testapp_python2_sqlite_openssl.py index 18ce7c4fcd..c1dcf53efc 100644 --- a/testapps/setup_testapp_python2_sqlite_openssl.py +++ b/testapps/setup_testapp_python2_sqlite_openssl.py @@ -5,7 +5,7 @@ options = {'apk': {'requirements': 'sdl2,pyjnius,kivy,python2,openssl,requests,peewee,sqlite3', 'android-api': 27, 'ndk-api': 21, - 'ndk-dir': '/home/sandy/android/crystax-ndk-10.3.2', + 'ndk-dir': '/home/sandy/android/android-ndk-r17c', 'dist-name': 'bdisttest_python2_sqlite_openssl', 'ndk-version': '10.3.2', 'permissions': ['INTERNET', 'VIBRATE'], diff --git a/testapps/setup_testapp_python3crystax.py b/testapps/setup_testapp_python3crystax.py deleted file mode 100644 index 08ed0afa09..0000000000 --- a/testapps/setup_testapp_python3crystax.py +++ /dev/null @@ -1,30 +0,0 @@ - -from distutils.core import setup -from setuptools import find_packages - -options = {'apk': {'requirements': 'sdl2,pyjnius,kivy,python3crystax', - 'android-api': 19, - 'ndk-api': 19, - 'ndk-dir': '/home/asandy/android/crystax-ndk-10.3.2', - 'dist-name': 'bdisttest_python3', - 'ndk-version': '10.3.2', - 'permission': 'VIBRATE', - }} - -package_data = {'': ['*.py', - '*.png'] - } - -packages = find_packages() -print('packages are', packages) - -setup( - name='testapp_python3', - version='1.1', - description='p4a setup.py test', - author='Alexander Taylor', - author_email='alexanderjohntaylor@gmail.com', - packages=find_packages(), - options=options, - package_data={'testapp': ['*.py', '*.png']} -) diff --git a/testapps/setup_vispy.py b/testapps/setup_vispy.py index a0863d0a1c..49ad47fda3 100644 --- a/testapps/setup_vispy.py +++ b/testapps/setup_vispy.py @@ -7,7 +7,7 @@ 'blacklist-requirements': 'openssl,sqlite3', 'android-api': 27, 'ndk-api': 21, - 'ndk-dir': '/home/asandy/android/crystax-ndk-10.3.2', + 'ndk-dir': '/home/asandy/android/android-ndk-r17c', 'dist-name': 'bdisttest', 'ndk-version': '10.3.2', 'permission': 'VIBRATE', diff --git a/tests/test_androidmodule_ctypes_finder.py b/tests/test_androidmodule_ctypes_finder.py index 7d4526888d..553287d12a 100644 --- a/tests/test_androidmodule_ctypes_finder.py +++ b/tests/test_androidmodule_ctypes_finder.py @@ -1,6 +1,12 @@ -import mock -from mock import MagicMock +# This test is still expected to support Python 2, as it tests +# on-Android functionality that we still maintain +try: # Python 3+ + from unittest import mock + from unittest.mock import MagicMock +except ImportError: # Python 2 + import mock + from mock import MagicMock import os import shutil import sys diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index f8f625132d..6f66925818 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -1,6 +1,6 @@ + import os import sh - import unittest try: @@ -9,12 +9,16 @@ # `Python 2` or lower than `Python 3.3` does not # have the `unittest.mock` module built-in import mock -from pythonforandroid.bootstrap import Bootstrap +from pythonforandroid.bootstrap import ( + _cmp_bootstraps_by_priority, Bootstrap, expand_dependencies, +) from pythonforandroid.distribution import Distribution from pythonforandroid.recipe import Recipe from pythonforandroid.archs import ArchARMv7_a from pythonforandroid.build import Context +from test_graph import get_fake_recipe + class BaseClassSetupBootstrap(object): """ @@ -90,7 +94,7 @@ def test_build_dist_dirs(self): - :meth:`~pythonforandroid.bootstrap.Bootstrap.get_dist_dir` - :meth:`~pythonforandroid.bootstrap.Bootstrap.get_common_dir` """ - bs = Bootstrap().get_bootstrap("sdl2", self.ctx) + bs = Bootstrap.get_bootstrap("sdl2", self.ctx) self.assertTrue( bs.get_build_dir().endswith("build/bootstrap_builds/sdl2-python3") @@ -100,32 +104,146 @@ def test_build_dist_dirs(self): bs.get_common_dir().endswith("pythonforandroid/bootstraps/common") ) - def test_list_bootstraps(self): + def test__cmp_bootstraps_by_priority(self): + # Test service_only has higher priority than sdl2: + # (higher priority = smaller number/comes first) + self.assertTrue(_cmp_bootstraps_by_priority( + Bootstrap.get_bootstrap("service_only", self.ctx), + Bootstrap.get_bootstrap("sdl2", self.ctx) + ) < 0) + + # Test a random bootstrap is always lower priority than sdl2: + class _FakeBootstrap(object): + def __init__(self, name): + self.name = name + bs1 = _FakeBootstrap("alpha") + bs2 = _FakeBootstrap("zeta") + self.assertTrue(_cmp_bootstraps_by_priority( + bs1, + Bootstrap.get_bootstrap("sdl2", self.ctx) + ) > 0) + self.assertTrue(_cmp_bootstraps_by_priority( + bs2, + Bootstrap.get_bootstrap("sdl2", self.ctx) + ) > 0) + + # Test bootstraps that aren't otherwise recognized are ranked + # alphabetically: + self.assertTrue(_cmp_bootstraps_by_priority( + bs2, + bs1, + ) > 0) + self.assertTrue(_cmp_bootstraps_by_priority( + bs1, + bs2, + ) < 0) + + def test_all_bootstraps(self): """A test which will initialize a bootstrap and will check if the method :meth:`~pythonforandroid.bootstrap.Bootstrap.list_bootstraps` returns the expected values, which should be: `empty", `service_only`, `webview` and `sdl2` """ expected_bootstraps = {"empty", "service_only", "webview", "sdl2"} - set_of_bootstraps = set(Bootstrap().list_bootstraps()) + set_of_bootstraps = Bootstrap.all_bootstraps() self.assertEqual( expected_bootstraps, expected_bootstraps & set_of_bootstraps ) self.assertEqual(len(expected_bootstraps), len(set_of_bootstraps)) + def test_expand_dependencies(self): + # Test dependency expansion of a recipe with no alternatives: + expanded_result_1 = expand_dependencies(["pysdl2"], self.ctx) + self.assertTrue( + {"sdl2", "pysdl2", "python3"} in + [set(s) for s in expanded_result_1] + ) + + # Test expansion of a single element but as tuple: + expanded_result_1 = expand_dependencies([("pysdl2",)], self.ctx) + self.assertTrue( + {"sdl2", "pysdl2", "python3"} in + [set(s) for s in expanded_result_1] + ) + + # Test all alternatives are listed (they won't have dependencies + # expanded since expand_dependencies() is too simplistic): + expanded_result_2 = expand_dependencies([("pysdl2", "kivy")], self.ctx) + self.assertEqual([["pysdl2"], ["kivy"]], expanded_result_2) + + def test_expand_dependencies_with_pure_python_package(self): + """Check that `expanded_dependencies`, with a pure python package as + one of the dependencies, returns a list of dependencies + """ + expanded_result = expand_dependencies( + ["python3", "kivy", "peewee"], self.ctx + ) + # we expect to have two results (one for python2 and one for python3) + self.assertEqual(len(expanded_result), 2) + self.assertIsInstance(expanded_result, list) + for i in expanded_result: + self.assertIsInstance(i, list) + def test_get_bootstraps_from_recipes(self): """A test which will initialize a bootstrap and will check if the method :meth:`~pythonforandroid.bootstrap.Bootstrap. get_bootstraps_from_recipes` returns the expected values """ + + import pythonforandroid.recipe + original_get_recipe = pythonforandroid.recipe.Recipe.get_recipe + + # Test that SDL2 works with kivy: recipes_sdl2 = {"sdl2", "python3", "kivy"} - bs = Bootstrap().get_bootstrap_from_recipes(recipes_sdl2, self.ctx) + bs = Bootstrap.get_bootstrap_from_recipes(recipes_sdl2, self.ctx) + self.assertEqual(bs.name, "sdl2") + # Test that pysdl2 or kivy alone will also yield SDL2 (dependency): + recipes_pysdl2_only = {"pysdl2"} + bs = Bootstrap.get_bootstrap_from_recipes( + recipes_pysdl2_only, self.ctx + ) + self.assertEqual(bs.name, "sdl2") + recipes_kivy_only = {"kivy"} + bs = Bootstrap.get_bootstrap_from_recipes( + recipes_kivy_only, self.ctx + ) self.assertEqual(bs.name, "sdl2") - # test wrong recipes + with mock.patch("pythonforandroid.recipe.Recipe.get_recipe") as \ + mock_get_recipe: + # Test that something conflicting with sdl2 won't give sdl2: + def _add_sdl2_conflicting_recipe(name, ctx): + if name == "conflictswithsdl2": + if name not in pythonforandroid.recipe.Recipe.recipes: + pythonforandroid.recipe.Recipe.recipes[name] = ( + get_fake_recipe("sdl2", conflicts=["sdl2"]) + ) + return original_get_recipe(name, ctx) + mock_get_recipe.side_effect = _add_sdl2_conflicting_recipe + recipes_with_sdl2_conflict = {"python3", "conflictswithsdl2"} + bs = Bootstrap.get_bootstrap_from_recipes( + recipes_with_sdl2_conflict, self.ctx + ) + self.assertNotEqual(bs.name, "sdl2") + + # Test using flask will default to webview: + recipes_with_flask = {"python3", "flask"} + bs = Bootstrap.get_bootstrap_from_recipes( + recipes_with_flask, self.ctx + ) + self.assertEqual(bs.name, "webview") + + # Test using random packages will default to service_only: + recipes_with_no_sdl2_or_web = {"python3", "numpy"} + bs = Bootstrap.get_bootstrap_from_recipes( + recipes_with_no_sdl2_or_web, self.ctx + ) + self.assertEqual(bs.name, "service_only") + + # Test wrong recipes wrong_recipes = {"python2", "python3", "pyjnius"} - bs = Bootstrap().get_bootstrap_from_recipes(wrong_recipes, self.ctx) + bs = Bootstrap.get_bootstrap_from_recipes(wrong_recipes, self.ctx) self.assertIsNone(bs) @mock.patch("pythonforandroid.bootstrap.ensure_dir") diff --git a/tests/test_distribution.py b/tests/test_distribution.py index 1716060513..dece511074 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -84,11 +84,12 @@ def test_folder_exist(self, mock_exists): :meth:`~pythonforandroid.distribution.Distribution.folder_exist` is called once with the proper arguments.""" + mock_exists.return_value = False self.setUp_distribution_with_bootstrap( - Bootstrap().get_bootstrap("sdl2", self.ctx) + Bootstrap.get_bootstrap("sdl2", self.ctx) ) self.ctx.bootstrap.distribution.folder_exists() - mock_exists.assert_called_once_with( + mock_exists.assert_called_with( self.ctx.bootstrap.distribution.dist_dir ) diff --git a/tests/test_entrypoints_python2.py b/tests/test_entrypoints_python2.py new file mode 100644 index 0000000000..0a2f6ebb4f --- /dev/null +++ b/tests/test_entrypoints_python2.py @@ -0,0 +1,38 @@ + +# This test is a special case that we expect to run under Python 2, so +# include the necessary compatibility imports: +try: # Python 3 + from unittest import mock +except ImportError: # Python 2 + import mock + +from pythonforandroid.recommendations import PY2_ERROR_TEXT +from pythonforandroid import entrypoints + + +def test_main_python2(): + """Test that running under Python 2 leads to the build failing, while + running under a suitable version works fine. + + Note that this test must be run *using* Python 2 to truly test + that p4a can reach the Python version check before importing some + Python-3-only syntax and crashing. + """ + + # Under Python 2, we should get a normal control flow exception + # that is handled properly, not any other type of crash + handle_exception_path = 'pythonforandroid.entrypoints.handle_build_exception' + with mock.patch('sys.version_info') as fake_version_info, \ + mock.patch(handle_exception_path) as handle_build_exception: # noqa: E127 + + fake_version_info.major = 2 + fake_version_info.minor = 7 + + def check_python2_exception(exc): + """Check that the exception message is Python 2 specific.""" + assert exc.message == PY2_ERROR_TEXT + handle_build_exception.side_effect = check_python2_exception + + entrypoints.main() + + handle_build_exception.assert_called_once() diff --git a/tests/test_graph.py b/tests/test_graph.py index 0534d58290..ccade98561 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -19,14 +19,14 @@ Bootstrap.get_bootstrap('sdl2', ctx)] valid_combinations = list(product(name_sets, bootstraps)) valid_combinations.extend( - [(['python3crystax'], Bootstrap.get_bootstrap('sdl2', ctx)), - (['kivy', 'python3crystax'], Bootstrap.get_bootstrap('sdl2', ctx)), + [(['python3'], Bootstrap.get_bootstrap('sdl2', ctx)), + (['kivy', 'python3'], Bootstrap.get_bootstrap('sdl2', ctx)), (['flask'], Bootstrap.get_bootstrap('webview', ctx)), (['pysdl2'], None), # auto-detect bootstrap! important corner case ] ) invalid_combinations = [ - [['python2', 'python3crystax'], None], + [['python2', 'python3'], None], [['pysdl2', 'genericndkbuild'], None], ] invalid_combinations_simple = list(invalid_combinations) @@ -39,12 +39,11 @@ # non-tuple/non-ambiguous dependencies, e.g.: # # dependencies_1st = ["python2", "pillow"] -# dependencies_2nd = ["python3crystax", "pillow"] +# dependencies_2nd = ["python3", "pillow"] # # This however won't work: # # dependencies_1st = [("python2", "python3"), "pillow"] -# dependencies_2nd = [("python2legacy", "python3crystax"), "pillow"] # # (This is simply because the conflict checker doesn't resolve this to # keep the code ismple enough) diff --git a/tests/test_recipe.py b/tests/test_recipe.py index 8946c92247..9685b15d68 100644 --- a/tests/test_recipe.py +++ b/tests/test_recipe.py @@ -1,11 +1,34 @@ import os +import pytest import types import unittest import warnings +import mock +from backports import tempfile from pythonforandroid.build import Context from pythonforandroid.recipe import Recipe, import_recipe +def patch_logger(level): + return mock.patch('pythonforandroid.recipe.{}'.format(level)) + + +def patch_logger_info(): + return patch_logger('info') + + +def patch_logger_debug(): + return patch_logger('debug') + + +def patch_urlretrieve(): + return mock.patch('pythonforandroid.recipe.urlretrieve') + + +class DummyRecipe(Recipe): + pass + + class TestRecipe(unittest.TestCase): def test_recipe_dirs(self): @@ -58,3 +81,98 @@ def test_import_recipe(self): module = import_recipe(name, pathname) assert module is not None assert recorded_warnings == [] + + def test_download_if_necessary(self): + """ + Download should happen via `Recipe.download()` only if the recipe + specific environment variable is not set. + """ + # download should happen as the environment variable is not set + recipe = DummyRecipe() + with mock.patch.object(Recipe, 'download') as m_download: + recipe.download_if_necessary() + assert m_download.call_args_list == [mock.call()] + # after setting it the download should be skipped + env_var = 'P4A_test_recipe_DIR' + env_dict = {env_var: '1'} + with mock.patch.object(Recipe, 'download') as m_download, mock.patch.dict(os.environ, env_dict): + recipe.download_if_necessary() + assert m_download.call_args_list == [] + + def test_download_url_not_set(self): + """ + Verifies that no download happens when URL is not set. + """ + recipe = DummyRecipe() + with patch_logger_info() as m_info: + recipe.download() + assert m_info.call_args_list == [ + mock.call('Skipping test_recipe download as no URL is set')] + + @staticmethod + def get_dummy_python_recipe_for_download_tests(): + """ + Helper method for creating a test recipe used in download tests. + """ + recipe = DummyRecipe() + filename = 'Python-3.7.4.tgz' + url = 'https://www.python.org/ftp/python/3.7.4/{}'.format(filename) + recipe._url = url + recipe.ctx = Context() + return recipe, filename + + def test_download_url_is_set(self): + """ + Verifies the actual download gets triggered when the URL is set. + """ + recipe, filename = self.get_dummy_python_recipe_for_download_tests() + url = recipe.url + with ( + patch_logger_debug()) as m_debug, ( + mock.patch.object(Recipe, 'download_file')) as m_download_file, ( + mock.patch('pythonforandroid.recipe.sh.touch')) as m_touch, ( + tempfile.TemporaryDirectory()) as temp_dir: + recipe.ctx.setup_dirs(temp_dir) + recipe.download() + assert m_download_file.call_args_list == [mock.call(url, filename)] + assert m_debug.call_args_list == [ + mock.call( + 'Downloading test_recipe from ' + 'https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz')] + assert m_touch.call_count == 1 + + def test_download_file_scheme_https(self): + """ + Verifies `urlretrieve()` is being called on https downloads. + """ + recipe, filename = self.get_dummy_python_recipe_for_download_tests() + url = recipe.url + with ( + patch_urlretrieve()) as m_urlretrieve, ( + tempfile.TemporaryDirectory()) as temp_dir: + recipe.ctx.setup_dirs(temp_dir) + assert recipe.download_file(url, filename) == filename + assert m_urlretrieve.call_args_list == [ + mock.call(url, filename, mock.ANY) + ] + + def test_download_file_scheme_https_oserror(self): + """ + Checks `urlretrieve()` is being retried on `OSError`. + After a number of retries the exception is re-reaised. + """ + recipe, filename = self.get_dummy_python_recipe_for_download_tests() + url = recipe.url + with ( + patch_urlretrieve()) as m_urlretrieve, ( + mock.patch('pythonforandroid.recipe.time.sleep')) as m_sleep, ( + pytest.raises(OSError)), ( + tempfile.TemporaryDirectory()) as temp_dir: + recipe.ctx.setup_dirs(temp_dir) + m_urlretrieve.side_effect = OSError + assert recipe.download_file(url, filename) == filename + retry = 5 + expected_call_args_list = [mock.call(url, filename, mock.ANY)] * retry + assert m_urlretrieve.call_args_list == expected_call_args_list + expected_call_args_list = [mock.call(1)] * (retry - 1) + assert m_sleep.call_args_list == expected_call_args_list diff --git a/tests/test_recommendations.py b/tests/test_recommendations.py new file mode 100644 index 0000000000..649fb3b1f9 --- /dev/null +++ b/tests/test_recommendations.py @@ -0,0 +1,239 @@ +import unittest +from os.path import join +from sys import version as py_version + +from unittest import mock +from pythonforandroid.recommendations import ( + check_ndk_api, + check_ndk_version, + check_target_api, + read_ndk_version, + check_python_version, + MAX_NDK_VERSION, + RECOMMENDED_NDK_VERSION, + RECOMMENDED_TARGET_API, + MIN_NDK_API, + MIN_NDK_VERSION, + NDK_DOWNLOAD_URL, + ARMEABI_MAX_TARGET_API, + MIN_TARGET_API, + UNKNOWN_NDK_MESSAGE, + PARSE_ERROR_NDK_MESSAGE, + READ_ERROR_NDK_MESSAGE, + ENSURE_RIGHT_NDK_MESSAGE, + NDK_LOWER_THAN_SUPPORTED_MESSAGE, + UNSUPPORTED_NDK_API_FOR_ARMEABI_MESSAGE, + CURRENT_NDK_VERSION_MESSAGE, + RECOMMENDED_NDK_VERSION_MESSAGE, + TARGET_NDK_API_GREATER_THAN_TARGET_API_MESSAGE, + OLD_NDK_API_MESSAGE, + NEW_NDK_MESSAGE, + OLD_API_MESSAGE, + MIN_PYTHON_MAJOR_VERSION, + MIN_PYTHON_MINOR_VERSION, + PY2_ERROR_TEXT, + PY_VERSION_ERROR_TEXT, +) + +from pythonforandroid.util import BuildInterruptingException + +running_in_py2 = int(py_version[0]) < 3 + + +class TestRecommendations(unittest.TestCase): + """ + An inherited class of `unittest.TestCase`to test the module + :mod:`~pythonforandroid.recommendations`. + """ + + def setUp(self): + self.ndk_dir = "/opt/android/android-ndk" + + @unittest.skipIf(running_in_py2, "`assertLogs` requires Python 3.4+") + @mock.patch("pythonforandroid.recommendations.read_ndk_version") + def test_check_ndk_version_greater_than_recommended(self, mock_read_ndk): + mock_read_ndk.return_value.version = [MAX_NDK_VERSION + 1, 0, 5232133] + with self.assertLogs(level="INFO") as cm: + check_ndk_version(self.ndk_dir) + mock_read_ndk.assert_called_once_with(self.ndk_dir) + self.assertEqual( + cm.output, + [ + "INFO:p4a:[INFO]: {}".format( + CURRENT_NDK_VERSION_MESSAGE.format( + ndk_version=MAX_NDK_VERSION + 1 + ) + ), + "WARNING:p4a:[WARNING]: {}".format( + RECOMMENDED_NDK_VERSION_MESSAGE.format( + recommended_ndk_version=RECOMMENDED_NDK_VERSION + ) + ), + "WARNING:p4a:[WARNING]: {}".format(NEW_NDK_MESSAGE), + ], + ) + + @mock.patch("pythonforandroid.recommendations.read_ndk_version") + def test_check_ndk_version_lower_than_recommended(self, mock_read_ndk): + mock_read_ndk.return_value.version = [MIN_NDK_VERSION - 1, 0, 5232133] + with self.assertRaises(BuildInterruptingException) as e: + check_ndk_version(self.ndk_dir) + self.assertEqual( + e.exception.args[0], + NDK_LOWER_THAN_SUPPORTED_MESSAGE.format( + min_supported=MIN_NDK_VERSION, ndk_url=NDK_DOWNLOAD_URL + ), + ) + mock_read_ndk.assert_called_once_with(self.ndk_dir) + + @unittest.skipIf(running_in_py2, "`assertLogs` requires Python 3.4+") + def test_check_ndk_version_error(self): + """ + Test that a fake ndk dir give us two messages: + - first should be an `INFO` log + - second should be an `WARNING` log + """ + with self.assertLogs(level="INFO") as cm: + check_ndk_version(self.ndk_dir) + self.assertEqual( + cm.output, + [ + "INFO:p4a:[INFO]: {}".format(UNKNOWN_NDK_MESSAGE), + "WARNING:p4a:[WARNING]: {}".format( + READ_ERROR_NDK_MESSAGE.format(ndk_dir=self.ndk_dir) + ), + "WARNING:p4a:[WARNING]: {}".format( + ENSURE_RIGHT_NDK_MESSAGE.format( + min_supported=MIN_NDK_VERSION, + rec_version=RECOMMENDED_NDK_VERSION, + ndk_url=NDK_DOWNLOAD_URL, + ) + ), + ], + ) + + @mock.patch("pythonforandroid.recommendations.open") + def test_read_ndk_version(self, mock_open_src_prop): + mock_open_src_prop.side_effect = [ + mock.mock_open( + read_data="Pkg.Revision = 17.2.4988734" + ).return_value + ] + version = read_ndk_version(self.ndk_dir) + mock_open_src_prop.assert_called_once_with( + join(self.ndk_dir, "source.properties") + ) + assert version == "17.2.4988734" + + @unittest.skipIf(running_in_py2, "`assertLogs` requires Python 3.4+") + @mock.patch("pythonforandroid.recommendations.open") + def test_read_ndk_version_error(self, mock_open_src_prop): + mock_open_src_prop.side_effect = [ + mock.mock_open(read_data="").return_value + ] + with self.assertLogs(level="INFO") as cm: + version = read_ndk_version(self.ndk_dir) + self.assertEqual( + cm.output, + ["INFO:p4a:[INFO]: {}".format(PARSE_ERROR_NDK_MESSAGE)], + ) + mock_open_src_prop.assert_called_once_with( + join(self.ndk_dir, "source.properties") + ) + assert version is None + + def test_check_target_api_error_arch_armeabi(self): + + with self.assertRaises(BuildInterruptingException) as e: + check_target_api(RECOMMENDED_TARGET_API, "armeabi") + self.assertEqual( + e.exception.args[0], + UNSUPPORTED_NDK_API_FOR_ARMEABI_MESSAGE.format( + req_ndk_api=RECOMMENDED_TARGET_API, + max_ndk_api=ARMEABI_MAX_TARGET_API, + ), + ) + + @unittest.skipIf(running_in_py2, "`assertLogs` requires Python 3.4+") + def test_check_target_api_warning_target_api(self): + + with self.assertLogs(level="INFO") as cm: + check_target_api(MIN_TARGET_API - 1, MIN_TARGET_API) + self.assertEqual( + cm.output, + [ + "WARNING:p4a:[WARNING]: Target API 25 < 26", + "WARNING:p4a:[WARNING]: {old_api_msg}".format( + old_api_msg=OLD_API_MESSAGE + ), + ], + ) + + def test_check_ndk_api_error_android_api(self): + """ + Given an `android api` greater than an `ndk_api`, we should get an + `BuildInterruptingException`. + """ + ndk_api = MIN_NDK_API + 1 + android_api = MIN_NDK_API + with self.assertRaises(BuildInterruptingException) as e: + check_ndk_api(ndk_api, android_api) + self.assertEqual( + e.exception.args[0], + TARGET_NDK_API_GREATER_THAN_TARGET_API_MESSAGE.format( + ndk_api=ndk_api, android_api=android_api + ), + ) + + @unittest.skipIf(running_in_py2, "`assertLogs` requires Python 3.4+") + def test_check_ndk_api_warning_old_ndk(self): + """ + Given an `android api` lower than the supported by p4a, we should + get an `BuildInterruptingException`. + """ + ndk_api = MIN_NDK_API - 1 + android_api = RECOMMENDED_TARGET_API + with self.assertLogs(level="INFO") as cm: + check_ndk_api(ndk_api, android_api) + self.assertEqual( + cm.output, + [ + "WARNING:p4a:[WARNING]: {}".format( + OLD_NDK_API_MESSAGE.format(MIN_NDK_API) + ) + ], + ) + + def test_check_python_version(self): + """With any version info lower than the minimum, we should get a + BuildInterruptingException with an appropriate message. + """ + with mock.patch('sys.version_info') as fake_version_info: + + # Major version is Python 2 => exception + fake_version_info.major = MIN_PYTHON_MAJOR_VERSION - 1 + fake_version_info.minor = MIN_PYTHON_MINOR_VERSION + with self.assertRaises(BuildInterruptingException) as context: + check_python_version() + assert context.exception.message == PY2_ERROR_TEXT + + # Major version too low => exception + # Using a float valued major version just to test the logic and avoid + # clashing with the Python 2 check + fake_version_info.major = MIN_PYTHON_MAJOR_VERSION - 0.1 + fake_version_info.minor = MIN_PYTHON_MINOR_VERSION + with self.assertRaises(BuildInterruptingException) as context: + check_python_version() + assert context.exception.message == PY_VERSION_ERROR_TEXT + + # Minor version too low => exception + fake_version_info.major = MIN_PYTHON_MAJOR_VERSION + fake_version_info.minor = MIN_PYTHON_MINOR_VERSION - 1 + with self.assertRaises(BuildInterruptingException) as context: + check_python_version() + assert context.exception.message == PY_VERSION_ERROR_TEXT + + # Version high enough => nothing interesting happens + fake_version_info.major = MIN_PYTHON_MAJOR_VERSION + fake_version_info.minor = MIN_PYTHON_MINOR_VERSION + check_python_version() diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py new file mode 100644 index 0000000000..11e35ff989 --- /dev/null +++ b/tests/test_toolchain.py @@ -0,0 +1,135 @@ +import io +import sys +import pytest +import mock +from pythonforandroid.recipe import Recipe +from pythonforandroid.toolchain import ToolchainCL +from pythonforandroid.util import BuildInterruptingException + + +def patch_sys_argv(argv): + return mock.patch('sys.argv', argv) + + +def patch_argparse_print_help(): + return mock.patch('argparse.ArgumentParser.print_help') + + +def patch_sys_stdout(): + return mock.patch('sys.stdout', new_callable=io.StringIO) + + +def raises_system_exit(): + return pytest.raises(SystemExit) + + +class TestToolchainCL: + + def test_help(self): + """ + Calling with `--help` should print help and exit 0. + """ + argv = ['toolchain.py', '--help', '--storage-dir=/tmp'] + with patch_sys_argv(argv), raises_system_exit( + ) as ex_info, patch_argparse_print_help() as m_print_help: + ToolchainCL() + assert ex_info.value.code == 0 + assert m_print_help.call_args_list == [mock.call()] + + @pytest.mark.skipif(sys.version_info < (3, 0), reason="requires python3") + def test_unknown(self): + """ + Calling with unknown args should print help and exit 1. + """ + argv = ['toolchain.py', '--unknown'] + with patch_sys_argv(argv), raises_system_exit( + ) as ex_info, patch_argparse_print_help() as m_print_help: + ToolchainCL() + assert ex_info.value.code == 1 + assert m_print_help.call_args_list == [mock.call()] + + def test_create(self): + """ + Basic `create` distribution test. + """ + argv = [ + 'toolchain.py', + 'create', + '--sdk-dir=/tmp/android-sdk', + '--ndk-dir=/tmp/android-ndk', + '--bootstrap=service_only', + '--requirements=python3', + '--dist-name=test_toolchain', + ] + with patch_sys_argv(argv), mock.patch( + 'pythonforandroid.build.get_available_apis' + ) as m_get_available_apis, mock.patch( + 'pythonforandroid.build.get_toolchain_versions' + ) as m_get_toolchain_versions, mock.patch( + 'pythonforandroid.build.get_ndk_platform_dir' + ) as m_get_ndk_platform_dir, mock.patch( + 'pythonforandroid.toolchain.build_recipes' + ) as m_build_recipes, mock.patch( + 'pythonforandroid.bootstraps.service_only.' + 'ServiceOnlyBootstrap.run_distribute' + ) as m_run_distribute: + m_get_available_apis.return_value = [27] + m_get_toolchain_versions.return_value = (['4.9'], True) + m_get_ndk_platform_dir.return_value = ( + '/tmp/android-ndk/platforms/android-21/arch-arm', True) + ToolchainCL() + assert m_get_available_apis.call_args_list == [ + mock.call('/tmp/android-sdk')] + assert m_get_toolchain_versions.call_args_list == [ + mock.call('/tmp/android-ndk', mock.ANY)] + build_order = [ + 'hostpython3', 'libffi', 'openssl', 'sqlite3', 'python3', + 'genericndkbuild', 'setuptools', 'six', 'pyjnius', 'android', + ] + python_modules = [] + context = mock.ANY + project_dir = None + assert m_build_recipes.call_args_list == [ + mock.call( + build_order, + python_modules, + context, + project_dir, + ignore_project_setup_py=False + ) + ] + assert m_run_distribute.call_args_list == [mock.call()] + + def test_create_no_sdk_dir(self): + """ + The `--sdk-dir` is mandatory to `create` a distribution. + """ + argv = ['toolchain.py', 'create'] + with patch_sys_argv(argv), pytest.raises( + BuildInterruptingException + ) as ex_info: + ToolchainCL() + assert ex_info.value.message == ( + 'Android SDK dir was not specified, exiting.') + + @pytest.mark.skipif(sys.version_info < (3, 0), reason="requires python3") + def test_recipes(self): + """ + Checks the `recipes` command prints out recipes information without crashing. + """ + argv = ['toolchain.py', 'recipes'] + with patch_sys_argv(argv), patch_sys_stdout() as m_stdout: + ToolchainCL() + # check if we have common patterns in the output + expected_strings = ( + 'conflicts:', + 'depends:', + 'kivy', + 'optional depends:', + 'python3', + 'sdl2', + ) + for expected_string in expected_strings: + assert expected_string in m_stdout.getvalue() + # deletes static attribute to not mess with other tests + del Recipe.recipes diff --git a/tox.ini b/tox.ini index 63393eab3f..2ca84ae803 100644 --- a/tox.ini +++ b/tox.ini @@ -8,13 +8,20 @@ deps = pytest virtualenv py3: coveralls -# makes it possible to override pytest args, e.g. -# tox -- tests/test_graph.py + backports.tempfile +# posargs will be replaced by the tox args, so you can override pytest +# args e.g. `tox -- tests/test_graph.py` commands = pytest {posargs:tests/} passenv = TRAVIS TRAVIS_* setenv = PYTHONPATH={toxinidir} +[testenv:py27] +# Note that the set of tests is not posargs-configurable here: we only +# check a minimal set of Python 2 tests for the remaining Python 2 +# functionality that we support +commands = pytest tests/test_androidmodule_ctypes_finder.py tests/test_entrypoints_python2.py + [testenv:py3] # for py3 env we will get code coverage commands = @@ -27,8 +34,11 @@ commands = flake8 pythonforandroid/ tests/ ci/ [flake8] ignore = - E123, E124, E126, - E226, - E402, E501, - W503, - W504 + E123, # Closing bracket does not match indentation of opening bracket's line + E124, # Closing bracket does not match visual indentation + E126, # Continuation line over-indented for hanging indent + E226, # Missing whitespace around arithmetic operator + E402, # Module level import not at top of file + E501, # Line too long (82 > 79 characters) + W503, # Line break occurred before a binary operator + W504 # Line break occurred after a binary operator