From 56e244f425d041d51bd9255c28d699e591cf8838 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 16 Nov 2019 15:18:58 +0100 Subject: [PATCH] Rework tests to isolate the currently reused wheelhouse directory utils is now a fixture providing tmp_path to run cibuildwheel --- bin/run_test.py | 5 +- test/01_basic/cibuildwheel_test.py | 15 +- test/02_test/cibuildwheel_test.py | 30 ++- test/03_before_build/cibuildwheel_test.py | 14 +- test/04_build_skip/cibuildwheel_test.py | 12 +- test/05_environment/cibuildwheel_test.py | 12 +- test/06_docker_images/cibuildwheel_test.py | 12 +- test/07_ssl/cibuildwheel_test.py | 11 +- .../cibuildwheel_test.py | 16 +- test/shared/utils.py | 208 ++++++++++-------- 10 files changed, 180 insertions(+), 155 deletions(-) diff --git a/bin/run_test.py b/bin/run_test.py index 4891d7839..33e8258f5 100755 --- a/bin/run_test.py +++ b/bin/run_test.py @@ -21,9 +21,6 @@ def single_run(test_project): env=env, ) - # clean up - if os.path.exists('wheelhouse'): - shutil.rmtree('wheelhouse') if __name__ == '__main__': import argparse @@ -33,7 +30,7 @@ def single_run(test_project): args = parser.parse_args() project_path = os.path.abspath(args.test_project_dir) - + if not os.path.exists(project_path): print('No test project not found.', file=sys.stderr) exit(2) diff --git a/test/01_basic/cibuildwheel_test.py b/test/01_basic/cibuildwheel_test.py index 998b55de5..388303809 100644 --- a/test/01_basic/cibuildwheel_test.py +++ b/test/01_basic/cibuildwheel_test.py @@ -1,20 +1,21 @@ import os -import utils +from utils import utils -project_dir = os.path.dirname(__file__) +PROJECT_DIR = os.path.dirname(__file__) -def test(): + +def test(utils): # build the wheels - utils.cibuildwheel_run(project_dir) + utils.cibuildwheel_run(PROJECT_DIR) # check that the expected wheels are produced expected_wheels = utils.expected_wheels('spam', '0.1.0') - actual_wheels = os.listdir('wheelhouse') + actual_wheels = utils.list_wheels() assert set(actual_wheels) == set(expected_wheels) -def test_build_identifiers(): +def test_build_identifiers(utils): # check that the number of expected wheels matches the number of build # identifiers # after adding CIBW_MANYLINUX_IMAGE to support manylinux2010, there @@ -22,5 +23,5 @@ def test_build_identifiers(): # the expected wheels expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0') if not '-manylinux' in w or '-manylinux1' in w] - build_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir) + build_identifiers = utils.cibuildwheel_get_build_identifiers(PROJECT_DIR) assert len(expected_wheels) == len(build_identifiers) diff --git a/test/02_test/cibuildwheel_test.py b/test/02_test/cibuildwheel_test.py index 4be4c7404..f645223a8 100644 --- a/test/02_test/cibuildwheel_test.py +++ b/test/02_test/cibuildwheel_test.py @@ -1,12 +1,14 @@ import os, subprocess import pytest -import utils +from utils import utils -def test(): - project_dir = os.path.dirname(__file__) +PROJECT_DIR = os.path.dirname(__file__) + + +def test(utils): # build and test the wheels - utils.cibuildwheel_run(project_dir, add_env={ + utils.cibuildwheel_run(PROJECT_DIR, add_env={ 'CIBW_TEST_REQUIRES': 'nose', # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. @@ -16,15 +18,13 @@ def test(): # also check that we got the right wheels expected_wheels = utils.expected_wheels('spam', '0.1.0') - actual_wheels = os.listdir('wheelhouse') + actual_wheels = utils.list_wheels() assert set(actual_wheels) == set(expected_wheels) -def test_extras_require(): - project_dir = os.path.dirname(__file__) - +def test_extras_require(utils): # build and test the wheels - utils.cibuildwheel_run(project_dir, add_env={ + utils.cibuildwheel_run(PROJECT_DIR, add_env={ 'CIBW_TEST_EXTRAS': 'test', # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. @@ -34,22 +34,20 @@ def test_extras_require(): # also check that we got the right wheels expected_wheels = utils.expected_wheels('spam', '0.1.0') - actual_wheels = os.listdir('wheelhouse') + actual_wheels = utils.list_wheels() assert set(actual_wheels) == set(expected_wheels) -def test_failing_test(): +def test_failing_test(utils): '''Ensure a failing test causes cibuildwheel to error out and exit''' - project_dir = os.path.dirname(__file__) - with pytest.raises(subprocess.CalledProcessError): - utils.cibuildwheel_run(project_dir, add_env={ + utils.cibuildwheel_run(PROJECT_DIR, add_env={ 'CIBW_TEST_COMMAND': 'false', - # manylinux1 has a version of bash that's been shown to have + # manylinux1 has a version of bash that's been shown to have # problems with this, so let's check that. 'CIBW_MANYLINUX_I686_IMAGE': 'manylinux1', 'CIBW_MANYLINUX_X86_64_IMAGE': 'manylinux1', }) - assert len(os.listdir('wheelhouse')) + assert len(utils.list_wheels()) == 0 diff --git a/test/03_before_build/cibuildwheel_test.py b/test/03_before_build/cibuildwheel_test.py index a9d7614d4..f57c52275 100644 --- a/test/03_before_build/cibuildwheel_test.py +++ b/test/03_before_build/cibuildwheel_test.py @@ -1,18 +1,20 @@ import os -import utils +from utils import utils -def test(): - project_dir = os.path.dirname(__file__) +PROJECT_DIR = os.path.dirname(__file__) + + +def test(utils): # build the wheels - utils.cibuildwheel_run(project_dir, add_env={ + utils.cibuildwheel_run(PROJECT_DIR, add_env={ # write python version information to a temporary file, this is # checked in setup.py 'CIBW_BEFORE_BUILD': '''python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonexecutable.txt', 'w').write(sys.executable)"''', 'CIBW_BEFORE_BUILD_WINDOWS': '''python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonexecutable.txt', 'w').write(sys.executable)"''', }) - + # also check that we got the right wheels expected_wheels = utils.expected_wheels('spam', '0.1.0') - actual_wheels = os.listdir('wheelhouse') + actual_wheels = utils.list_wheels() assert set(actual_wheels) == set(expected_wheels) diff --git a/test/04_build_skip/cibuildwheel_test.py b/test/04_build_skip/cibuildwheel_test.py index 764874a2a..7b915e618 100644 --- a/test/04_build_skip/cibuildwheel_test.py +++ b/test/04_build_skip/cibuildwheel_test.py @@ -1,11 +1,13 @@ import os -import utils +from utils import utils -def test(): - project_dir = os.path.dirname(__file__) +PROJECT_DIR = os.path.dirname(__file__) + + +def test(utils): # build the wheels - utils.cibuildwheel_run(project_dir, add_env={ + utils.cibuildwheel_run(PROJECT_DIR, add_env={ 'CIBW_BUILD': 'cp3?-*', 'CIBW_SKIP': 'cp37-*', }) @@ -13,5 +15,5 @@ def test(): # check that we got the right wheels. There should be no 2.7 or 3.7. expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0') if ('-cp3' in w) and ('-cp37' not in w)] - actual_wheels = os.listdir('wheelhouse') + actual_wheels = utils.list_wheels() assert set(actual_wheels) == set(expected_wheels) diff --git a/test/05_environment/cibuildwheel_test.py b/test/05_environment/cibuildwheel_test.py index 3b0a0678a..897d5718b 100644 --- a/test/05_environment/cibuildwheel_test.py +++ b/test/05_environment/cibuildwheel_test.py @@ -1,18 +1,20 @@ import os -import utils +from utils import utils -def test(): - project_dir = os.path.dirname(__file__) +PROJECT_DIR = os.path.dirname(__file__) + + +def test(utils): # write some information into the CIBW_ENVIRONMENT, for expansion and # insertion into the environment by cibuildwheel. This is checked # in setup.py - utils.cibuildwheel_run(project_dir, add_env={ + utils.cibuildwheel_run(PROJECT_DIR, add_env={ 'CIBW_ENVIRONMENT': '''CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$(echo 'test string 3')" PATH=$PATH:/opt/cibw_test_path''', 'CIBW_ENVIRONMENT_WINDOWS': '''CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$(echo 'test string 3')" PATH="$PATH;/opt/cibw_test_path"''', }) # also check that we got the right wheels built expected_wheels = utils.expected_wheels('spam', '0.1.0') - actual_wheels = os.listdir('wheelhouse') + actual_wheels = utils.list_wheels() assert set(actual_wheels) == set(expected_wheels) diff --git a/test/06_docker_images/cibuildwheel_test.py b/test/06_docker_images/cibuildwheel_test.py index 9067bc0f8..6b9386e9f 100644 --- a/test/06_docker_images/cibuildwheel_test.py +++ b/test/06_docker_images/cibuildwheel_test.py @@ -1,13 +1,15 @@ import os, pytest -import utils +from utils import utils -def test(): - project_dir = os.path.dirname(__file__) +PROJECT_DIR = os.path.dirname(__file__) + + +def test(utils): if utils.platform != 'linux': pytest.skip('the test is only relevant to the linux build') - utils.cibuildwheel_run(project_dir, add_env={ + utils.cibuildwheel_run(PROJECT_DIR, add_env={ 'CIBW_MANYLINUX_X86_64_IMAGE': 'dockcross/manylinux2010-x64', 'CIBW_MANYLINUX_I686_IMAGE': 'dockcross/manylinux1-x86', 'CIBW_BEFORE_BUILD': '/opt/python/cp36-cp36m/bin/pip install -U auditwheel', # Currently necessary on dockcross images to get auditwheel 2.1 supporting AUDITWHEEL_PLAT @@ -17,5 +19,5 @@ def test(): # also check that we got the right wheels built expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0') if '-manylinux2010_i686' not in w] - actual_wheels = os.listdir('wheelhouse') + actual_wheels = utils.list_wheels() assert set(actual_wheels) == set(expected_wheels) diff --git a/test/07_ssl/cibuildwheel_test.py b/test/07_ssl/cibuildwheel_test.py index fd3fff872..4b5ca0881 100644 --- a/test/07_ssl/cibuildwheel_test.py +++ b/test/07_ssl/cibuildwheel_test.py @@ -1,9 +1,12 @@ import os -import utils +from utils import utils -def test(): - project_dir = os.path.dirname(__file__) + +PROJECT_DIR = os.path.dirname(__file__) + + +def test(utils): # this test checks that SSL is working in the build environment using # some checks in setup.py. - utils.cibuildwheel_run(project_dir) + utils.cibuildwheel_run(PROJECT_DIR) diff --git a/test/08_manylinux2010_only/cibuildwheel_test.py b/test/08_manylinux2010_only/cibuildwheel_test.py index dab763e71..0d2ade30d 100644 --- a/test/08_manylinux2010_only/cibuildwheel_test.py +++ b/test/08_manylinux2010_only/cibuildwheel_test.py @@ -1,21 +1,23 @@ import os, pytest -import utils +from utils import utils -def test(): - project_dir = os.path.dirname(__file__) +PROJECT_DIR = os.path.dirname(__file__) + + +def test(utils): if utils.platform != 'linux': pytest.skip('the docker test is only relevant to the linux build') # build the wheels # CFLAGS environment veriable is ecessary to fail on 'malloc_info' (on manylinux1) during compilation/linking, - # rather than when dynamically loading the Python - utils.cibuildwheel_run(project_dir, add_env={ + # rather than when dynamically loading the Python + utils.cibuildwheel_run(PROJECT_DIR, add_env={ 'CIBW_ENVIRONMENT': 'CFLAGS="$CFLAGS -Werror=implicit-function-declaration"', }) - + # also check that we got the right wheels expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0') if not '-manylinux' in w or '-manylinux2010' in w] - actual_wheels = os.listdir('wheelhouse') + actual_wheels = utils.list_wheels() assert set(actual_wheels) == set(expected_wheels) diff --git a/test/shared/utils.py b/test/shared/utils.py index d4e9c9c62..d2edb2db1 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -5,106 +5,12 @@ ''' import subprocess, sys, os +import pytest + IS_WINDOWS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') IS_WINDOWS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' - -def cibuildwheel_get_build_identifiers(project_path, env=None): - ''' - Returns the list of build identifiers that cibuildwheel will try to build - for the current platform. - ''' - cmd_output = subprocess.check_output( - [sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers', project_path], - universal_newlines=True, - env=env, - ) - - return cmd_output.strip().split('\n') - - -def cibuildwheel_run(project_path, env=None, add_env=None): - ''' - Runs cibuildwheel as a subprocess, building the project at project_path. - - Uses the current Python interpreter. - Configure settings using env. - ''' - if env is None: - env = os.environ.copy() - - if add_env is not None: - env.update(add_env) - - subprocess.check_call( - [sys.executable, '-m', 'cibuildwheel', project_path], - env=env, - ) - - -def expected_wheels(package_name, package_version): - ''' - Returns a list of expected wheels from a run of cibuildwheel. - ''' - if platform == 'linux': - templates = [ - '{package_name}-{package_version}-cp27-cp27m-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp27-cp27mu-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp35-cp35m-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp36-cp36m-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp37-cp37m-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp38-cp38-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp27-cp27m-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp27-cp27mu-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp35-cp35m-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp36-cp36m-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp37-cp37m-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp38-cp38-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp27-cp27m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp27-cp27mu-manylinux1_i686.whl', - '{package_name}-{package_version}-cp35-cp35m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp36-cp36m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp37-cp37m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp38-cp38-manylinux1_i686.whl', - '{package_name}-{package_version}-cp27-cp27m-manylinux2010_i686.whl', - '{package_name}-{package_version}-cp27-cp27mu-manylinux2010_i686.whl', - '{package_name}-{package_version}-cp35-cp35m-manylinux2010_i686.whl', - '{package_name}-{package_version}-cp36-cp36m-manylinux2010_i686.whl', - '{package_name}-{package_version}-cp37-cp37m-manylinux2010_i686.whl', - '{package_name}-{package_version}-cp38-cp38-manylinux2010_i686.whl', - ] - elif platform == 'windows': - templates = [ - '{package_name}-{package_version}-cp27-cp27m-win32.whl', - '{package_name}-{package_version}-cp35-cp35m-win32.whl', - '{package_name}-{package_version}-cp36-cp36m-win32.whl', - '{package_name}-{package_version}-cp37-cp37m-win32.whl', - '{package_name}-{package_version}-cp38-cp38-win32.whl', - '{package_name}-{package_version}-cp27-cp27m-win_amd64.whl', - '{package_name}-{package_version}-cp35-cp35m-win_amd64.whl', - '{package_name}-{package_version}-cp36-cp36m-win_amd64.whl', - '{package_name}-{package_version}-cp37-cp37m-win_amd64.whl', - '{package_name}-{package_version}-cp38-cp38-win_amd64.whl', - ] - elif platform == 'macos': - templates = [ - '{package_name}-{package_version}-cp27-cp27m-macosx_10_6_intel.whl', - '{package_name}-{package_version}-cp35-cp35m-macosx_10_6_intel.whl', - '{package_name}-{package_version}-cp36-cp36m-macosx_10_6_intel.whl', - '{package_name}-{package_version}-cp37-cp37m-macosx_10_6_intel.whl', - '{package_name}-{package_version}-cp38-cp38-macosx_10_9_x86_64.whl', - ] - else: - raise Exception('unsupported platform') - - if IS_WINDOWS_RUNNING_ON_TRAVIS: - # Python 2.7 isn't supported on Travis. - templates = [t for t in templates if '-cp27-' not in t] - - return [filename.format(package_name=package_name, package_version=package_version) - for filename in templates] - platform = None if 'CIBW_PLATFORM' in os.environ: @@ -117,3 +23,113 @@ def expected_wheels(package_name, package_version): platform = 'windows' else: raise Exception('Unsupported platform') + + +@pytest.fixture +def utils(tmp_path): + ''' + Fixture use to run cibuildwheel with outputdir in a tmp_path, also list the wheels in that tmp_path + ''' + class _utils: + + def cibuildwheel_get_build_identifiers(self, project_path, env=None): + ''' + Returns the list of build identifiers that cibuildwheel will try to build + for the current platform. + ''' + cmd_output = subprocess.check_output( + [sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers', project_path], + universal_newlines=True, + env=env, + ) + + return cmd_output.strip().split('\n') + + def cibuildwheel_run(self, project_path, env=None, add_env=None): + ''' + Runs cibuildwheel as a subprocess, building the project at project_path. + + Uses the current Python interpreter. + Configure settings using env. + ''' + if env is None: + env = os.environ.copy() + + if add_env is not None: + env.update(add_env) + + subprocess.check_call( + [sys.executable, '-m', 'cibuildwheel', '--output-dir', str(tmp_path), project_path], + env=env, + ) + + def expected_wheels(self, package_name, package_version): + ''' + Returns a list of expected wheels from a run of cibuildwheel. + ''' + if platform == 'linux': + templates = [ + '{package_name}-{package_version}-cp27-cp27m-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp27-cp27mu-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp35-cp35m-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp36-cp36m-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp37-cp37m-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp38-cp38-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp27-cp27m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp27-cp27mu-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp35-cp35m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp36-cp36m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp37-cp37m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp38-cp38-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp27-cp27m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp27-cp27mu-manylinux1_i686.whl', + '{package_name}-{package_version}-cp35-cp35m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp36-cp36m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp37-cp37m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp38-cp38-manylinux1_i686.whl', + '{package_name}-{package_version}-cp27-cp27m-manylinux2010_i686.whl', + '{package_name}-{package_version}-cp27-cp27mu-manylinux2010_i686.whl', + '{package_name}-{package_version}-cp35-cp35m-manylinux2010_i686.whl', + '{package_name}-{package_version}-cp36-cp36m-manylinux2010_i686.whl', + '{package_name}-{package_version}-cp37-cp37m-manylinux2010_i686.whl', + '{package_name}-{package_version}-cp38-cp38-manylinux2010_i686.whl', + ] + elif platform == 'windows': + templates = [ + '{package_name}-{package_version}-cp27-cp27m-win32.whl', + '{package_name}-{package_version}-cp35-cp35m-win32.whl', + '{package_name}-{package_version}-cp36-cp36m-win32.whl', + '{package_name}-{package_version}-cp37-cp37m-win32.whl', + '{package_name}-{package_version}-cp38-cp38-win32.whl', + '{package_name}-{package_version}-cp27-cp27m-win_amd64.whl', + '{package_name}-{package_version}-cp35-cp35m-win_amd64.whl', + '{package_name}-{package_version}-cp36-cp36m-win_amd64.whl', + '{package_name}-{package_version}-cp37-cp37m-win_amd64.whl', + '{package_name}-{package_version}-cp38-cp38-win_amd64.whl', + ] + elif platform == 'macos': + templates = [ + '{package_name}-{package_version}-cp27-cp27m-macosx_10_6_intel.whl', + '{package_name}-{package_version}-cp35-cp35m-macosx_10_6_intel.whl', + '{package_name}-{package_version}-cp36-cp36m-macosx_10_6_intel.whl', + '{package_name}-{package_version}-cp37-cp37m-macosx_10_6_intel.whl', + '{package_name}-{package_version}-cp38-cp38-macosx_10_9_x86_64.whl', + ] + else: + raise Exception('unsupported platform') + + if IS_WINDOWS_RUNNING_ON_TRAVIS: + # Python 2.7 isn't supported on Travis. + templates = [t for t in templates if '-cp27-' not in t] + + return [filename.format(package_name=package_name, package_version=package_version) + for filename in templates] + + def list_wheels(self): + return os.listdir(str(tmp_path)) + + @property + def platform(self): + return platform + + yield _utils()