From 16d38ad189522b5bbd87cecf3550d72d18f5ab7c Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 24 Apr 2020 15:45:36 +0200 Subject: [PATCH 1/6] :alien: Migrate to `toml` and apply `isort` to `pythonpackage.py` Because `pytoml` is deprecated See also: https://pypi.org/project/pytoml/ --- pythonforandroid/pythonpackage.py | 21 +++++++++------------ setup.py | 2 +- tests/test_pythonpackage_basic.py | 4 ++++ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pythonforandroid/pythonpackage.py b/pythonforandroid/pythonpackage.py index f96042d982..a6e1843794 100644 --- a/pythonforandroid/pythonpackage.py +++ b/pythonforandroid/pythonpackage.py @@ -33,12 +33,8 @@ """ -from io import open # needed for python 2 import functools import os -from pep517.envbuild import BuildEnvironment -from pep517.wrappers import Pep517HookCaller -import pytoml import shutil import subprocess import sys @@ -46,13 +42,14 @@ import tempfile import textwrap import time -try: - from urllib.parse import urlparse - from urllib.parse import unquote as urlunquote -except ImportError: # Python 2... - from urlparse import urlparse - from urlparse import unquote as urlunquote import zipfile +from io import open # needed for python 2 +from urllib.parse import unquote as urlunquote +from urllib.parse import urlparse + +import toml +from pep517.envbuild import BuildEnvironment +from pep517.wrappers import Pep517HookCaller def transform_dep_for_pip(dependency): @@ -486,7 +483,7 @@ def _extract_metainfo_files_from_package_unsafe( # Get build backend and requirements from pyproject.toml: with open(os.path.join(path, 'pyproject.toml')) as f: - build_sys = pytoml.load(f)['build-system'] + build_sys = toml.load(f)['build-system'] backend = build_sys["build-backend"] build_requires.extend(build_sys["requires"]) @@ -630,7 +627,7 @@ def _extract_info_from_package(dependency, ) and include_build_requirements: # Read build system from pyproject.toml file: (PEP518) with open(os.path.join(output_folder, 'pyproject.toml')) as f: - build_sys = pytoml.load(f)['build-system'] + build_sys = toml.load(f)['build-system'] if "requires" in build_sys: requirements += build_sys["requires"] elif include_build_requirements: diff --git a/setup.py b/setup.py index 19a77bb756..99a38b91b7 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ install_reqs = [ 'appdirs', 'colorama>=0.3.3', 'jinja2', 'six', 'enum34; python_version<"3.4"', 'sh>=1.10; sys_platform!="nt"', - 'pep517<0.7.0"', 'pytoml', 'virtualenv<20' + 'pep517<0.7.0"', 'toml', 'virtualenv<20' ] # (pep517, pytoml and virtualenv are used by pythonpackage.py) diff --git a/tests/test_pythonpackage_basic.py b/tests/test_pythonpackage_basic.py index a39d8a44e3..6e7c469fcc 100644 --- a/tests/test_pythonpackage_basic.py +++ b/tests/test_pythonpackage_basic.py @@ -340,6 +340,10 @@ def test_venv(self): os.path.join(test_dir, "venv", "bin", "pip"), "install", "-U", "pep517<0.7.0" ]) + subprocess.check_output([ + os.path.join(test_dir, "venv", "bin", "pip"), + "install", "-U", "toml" + ]) sys_python_path = self.run__get_system_python_executable( os.path.join(test_dir, "venv", "bin", "python") ) From d6bc038bcd857b991e2d8c9b2aa4d90e59887b72 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 24 Apr 2020 15:51:21 +0200 Subject: [PATCH 2/6] :bug: Avoid copying `.tox` folder when extracting metadata from package Because when running local tests for `test_pythonpackages` and we have some virtual envs created with tox the function also copies the hidden tox folder causing error on our tests --- pythonforandroid/pythonpackage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pythonforandroid/pythonpackage.py b/pythonforandroid/pythonpackage.py index a6e1843794..66dec4fc31 100644 --- a/pythonforandroid/pythonpackage.py +++ b/pythonforandroid/pythonpackage.py @@ -108,7 +108,8 @@ def extract_metainfo_files_from_package( if is_filesystem_path(package): shutil.copytree( parse_as_folder_reference(package), - os.path.join(temp_folder, "package") + os.path.join(temp_folder, "package"), + ignore=shutil.ignore_patterns(".tox") ) package = os.path.join(temp_folder, "package") From fdd66d3812c0facca7dc14f2105c3b53c0eb8a1c Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 24 Apr 2020 20:19:20 +0200 Subject: [PATCH 3/6] :fire: Use python3's venv instead of `virtualenv` --- .travis.yml | 4 +--- Dockerfile | 1 - Makefile | 2 +- pythonforandroid/build.py | 22 ++++------------- pythonforandroid/util.py | 12 ---------- setup.py | 2 +- tests/test_pythonpackage_basic.py | 38 ----------------------------- tests/test_util.py | 40 ------------------------------- tox.ini | 1 - 9 files changed, 8 insertions(+), 114 deletions(-) diff --git a/.travis.yml b/.travis.yml index 27680224fc..86b0035ccc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,9 +23,7 @@ jobs: # See also: https://github.com/travis-ci/travis-ci/issues/8589 - type -t deactivate && deactivate || true - export PATH=/opt/python/3.7/bin:$PATH - # Install tox & virtualenv - # Note: venv/virtualenv are both used by tests/test_pythonpackage.py - - pip3.7 install -U virtualenv + # Install tox - pip3.7 install tox>=2.0 # Install coveralls & dependencies # Note: pyOpenSSL needed to send the coveralls reports diff --git a/Dockerfile b/Dockerfile index a466c3ef7b..027dbd2893 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,7 +77,6 @@ RUN dpkg --add-architecture i386 \ python3-venv \ sudo \ unzip \ - virtualenv \ wget \ zip \ zlib1g-dev \ diff --git a/Makefile b/Makefile index 3858e537ae..df7cdac17b 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ ANDROID_NDK_HOME ?= $(HOME)/.android/android-ndk all: virtualenv $(VIRTUAL_ENV): - virtualenv --python=$(PYTHON_WITH_VERSION) $(VIRTUAL_ENV) + python3 -m venv $(VIRTUAL_ENV) $(PIP) install Cython==0.28.6 $(PIP) install -e . diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index e6b3f4ca44..d12181b698 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -13,8 +13,8 @@ import subprocess from pythonforandroid.util import ( - current_directory, ensure_dir, get_virtualenv_executable, - BuildInterruptingException + current_directory, ensure_dir, + BuildInterruptingException, ) from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint) from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86, Archx86_64 @@ -357,13 +357,6 @@ def prepare_build_environment(self, check_ndk_api(ndk_api, self.android_api) - virtualenv = get_virtualenv_executable() - if virtualenv is None: - raise IOError('Couldn\'t find a virtualenv executable, ' - 'you must install this to use p4a.') - self.virtualenv = virtualenv - info('Found virtualenv at {}'.format(virtualenv)) - # path to some tools self.ccache = sh.which("ccache") if not self.ccache: @@ -765,15 +758,10 @@ def run_pymodules_install(ctx, modules, project_dir=None, info('Will process project install, if it fails then the ' 'project may not be compatible for Android install.') - venv = sh.Command(ctx.virtualenv) + # Use our hostpython to create the virtualenv + host_python = sh.Command(ctx.hostpython) with current_directory(join(ctx.build_dir)): - shprint(venv, - '--python=python{}'.format( - ctx.python_recipe.major_minor_version_string. - partition(".")[0] - ), - 'venv' - ) + shprint(host_python, '-m', 'venv', 'venv') # Prepare base environment and upgrade pip: base_env = copy.copy(os.environ) diff --git a/pythonforandroid/util.py b/pythonforandroid/util.py index 80a607f5fe..c9b779829a 100644 --- a/pythonforandroid/util.py +++ b/pythonforandroid/util.py @@ -1,7 +1,6 @@ import contextlib from os.path import exists, join from os import getcwd, chdir, makedirs, walk, uname -import sh import shutil from fnmatch import fnmatch from tempfile import mkdtemp @@ -55,17 +54,6 @@ def ensure_dir(filename): makedirs(filename) -def get_virtualenv_executable(): - virtualenv = None - if virtualenv is None: - virtualenv = sh.which('virtualenv2') - if virtualenv is None: - virtualenv = sh.which('virtualenv-2.7') - if virtualenv is None: - virtualenv = sh.which('virtualenv') - return virtualenv - - def walk_valid_filens(base_dir, invalid_dir_names, invalid_file_patterns): """Recursively walks all the files and directories in ``dirn``, ignoring directories that match any pattern in ``invalid_dirns`` diff --git a/setup.py b/setup.py index 99a38b91b7..4dce5e83d0 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ install_reqs = [ 'appdirs', 'colorama>=0.3.3', 'jinja2', 'six', 'enum34; python_version<"3.4"', 'sh>=1.10; sys_platform!="nt"', - 'pep517<0.7.0"', 'toml', 'virtualenv<20' + 'pep517<0.7.0"', 'toml', ] # (pep517, pytoml and virtualenv are used by pythonpackage.py) diff --git a/tests/test_pythonpackage_basic.py b/tests/test_pythonpackage_basic.py index 6e7c469fcc..9e6596d69e 100644 --- a/tests/test_pythonpackage_basic.py +++ b/tests/test_pythonpackage_basic.py @@ -6,7 +6,6 @@ """ import os -import pytest import shutil import sys import subprocess @@ -279,43 +278,6 @@ def test_systemwide_python(self): else: raise - def test_virtualenv(self): - """ Verifies that _get_system_python_executable() works correctly - if called with a python binary as found inside a virtualenv. - """ - - # Get system-wide python bin seen from here first: - pybin = _get_system_python_executable() - # (this call was not a test, we really just need the path here) - - test_dir = tempfile.mkdtemp() - try: - # Check that in a virtualenv, the system-wide python is returned: - subprocess.check_output([ - pybin, "-m", "virtualenv", - "--python=" + str(sys.executable), - "--", - os.path.join(test_dir, "virtualenv") - ]) - subprocess.check_output([ - os.path.join(test_dir, "virtualenv", "bin", "pip"), - "install", "-U", "pip" - ]) - subprocess.check_output([ - os.path.join(test_dir, "virtualenv", "bin", "pip"), - "install", "-U", "pep517<0.7.0" - ]) - sys_python_path = self.run__get_system_python_executable( - os.path.join(test_dir, "virtualenv", "bin", "python") - ) - assert os.path.normpath(sys_python_path).startswith( - os.path.normpath(pybin) - ) - finally: - shutil.rmtree(test_dir) - - @pytest.mark.skipif(int(sys.version.partition(".")[0]) < 3, - reason="venv is python 3 only") def test_venv(self): """ Verifies that _get_system_python_executable() works correctly in a 'venv' (Python 3 only feature). diff --git a/tests/test_util.py b/tests/test_util.py index 0653991639..ff57dc7a47 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -70,46 +70,6 @@ def test_current_directory_exception(self): ): pass - @mock.patch("pythonforandroid.util.sh.which") - def test_get_virtualenv_executable(self, mock_sh_which): - """ - Test method :meth:`~pythonforandroid.util.get_virtualenv_executable`. - In here we test: - - - that all calls to `sh.which` are performed, so we expect the - first two `sh.which` calls should be None and the last one should - return the expected virtualenv (the python3 one) - - that we don't have virtualenv installed, so all calls to - `sh.which` should return None - """ - expected_venv = os.path.join( - os.path.expanduser("~"), ".local/bin/virtualenv" - ) - mock_sh_which.side_effect = [None, None, expected_venv] - self.assertEqual(util.get_virtualenv_executable(), expected_venv) - mock_sh_which.assert_has_calls( - [ - mock.call("virtualenv2"), - mock.call("virtualenv-2.7"), - mock.call("virtualenv"), - ] - ) - self.assertEqual(mock_sh_which.call_count, 3) - mock_sh_which.reset_mock() - - # Now test that we don't have virtualenv installed, so all calls to - # `sh.which` should return None - mock_sh_which.side_effect = [None, None, None] - self.assertIsNone(util.get_virtualenv_executable()) - self.assertEqual(mock_sh_which.call_count, 3) - mock_sh_which.assert_has_calls( - [ - mock.call("virtualenv2"), - mock.call("virtualenv-2.7"), - mock.call("virtualenv"), - ] - ) - @mock.patch("pythonforandroid.util.walk") def test_walk_valid_filens(self, mock_walk): """ diff --git a/tox.ini b/tox.ini index 9ded48b328..4e69dd89b6 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,6 @@ basepython = python3 [testenv] deps = pytest - virtualenv py3: coveralls backports.tempfile # posargs will be replaced by the tox args, so you can override pytest From e5207353e759aa0a4a117f16c2dfc8ecacb1b2bb Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 24 Apr 2020 21:20:06 +0200 Subject: [PATCH 4/6] :green_heart: Add new dependency to docker `libssl-dev` So this way we will get hostpython's _ssl module build, and we will be able to use hostpython's pip without security errors. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 027dbd2893..07377e4248 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,6 +66,7 @@ RUN dpkg --add-architecture i386 \ libncurses5:i386 \ libpangox-1.0-0:i386 \ libpangoxft-1.0-0:i386 \ + libssl-dev \ libstdc++6:i386 \ libtool \ openjdk-8-jdk \ From ab2e9429457373ea908068e81ee9e004afaada25 Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 25 Apr 2020 16:36:00 +0200 Subject: [PATCH 5/6] :pencil: Update outdated inline comment --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4dce5e83d0..2814e41fab 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ 'enum34; python_version<"3.4"', 'sh>=1.10; sys_platform!="nt"', 'pep517<0.7.0"', 'toml', ] -# (pep517, pytoml and virtualenv are used by pythonpackage.py) +# (pep517 and toml are used by pythonpackage.py) # By specifying every file manually, package_data will be able to # include them in binary distributions. Note that we have to add From 9b83258b3adfc00491a1248728858b162451cbb9 Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 25 Apr 2020 16:44:37 +0200 Subject: [PATCH 6/6] :apple: Don't run p4a inside virtualenv/venv because it seems that hostpython's build scripts gets confused on about locating the `pyconfig.h` file **Note:** This only happens to macOS platform --- .travis.yml | 2 +- Makefile | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 86b0035ccc..04e49e59c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,7 +52,7 @@ jobs: # installs java 1.8, android's SDK/NDK and p4a - make -f ci/makefiles/osx.mk - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home - script: make testapps/armeabi-v7a PYTHON_WITH_VERSION=python3 + script: make testapps-no-venv/armeabi-v7a - <<: *testapps name: Rebuild updated recipes script: travis_wait 30 make docker/run/make/rebuild_updated_recipes diff --git a/Makefile b/Makefile index df7cdac17b..ef0c503eb9 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,14 @@ testapps/%: virtualenv python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ --arch=$($@_APP_ARCH) +testapps-no-venv/%: + pip3 install Cython==0.28.6 + pip3 install -e . + $(eval $@_APP_ARCH := $(shell basename $*)) + cd testapps/on_device_unit_tests/ && \ + python3 setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + --arch=$($@_APP_ARCH) + clean: find . -type d -name "__pycache__" -exec rm -r {} + find . -type d -name "*.egg-info" -exec rm -r {} +