From e0d7ef2f1766cdf65094c2b29bb6b40307277228 Mon Sep 17 00:00:00 2001 From: Tyler Reddy Date: Wed, 29 Apr 2020 23:48:00 -0600 Subject: [PATCH] MAINT: init file and mpl updates (#70) * MAINT: init file and mpl updates * we've been getting reports of new Python 3.7 patch releases from Windows store causing SciPy DLL load issues: https://github.com/scipy/scipy/issues/11826 * so, extend the `_distributor_init.py` machinery usage to include Python 3.7 and up for our wheels * simplify the appveyor `matplotlib` install now that there are stable releases available for Python `3.8` * MAINT: update .travis.yml * fix warnings in `.travis.yml` based on feedback from https://config.travis-ci.com/explore * bump distro to `bionic` since NumPy did this recently too * attempting to restore Travis CI runs to our wheels build matrix * MAINT: revise PR 70 * try using `[System.Version]` for more robust Python version comparisons in powershell * MAINT: PR 70 revisions * update minimum NumPy version to `1.14.5` * the distutils override for appveyor/Windows builds now only happens for Python < 3.7, since we have now added the `_distributor_init.py` for Python 3.7 (not just Python 3.8) * DEBUG: try newer NumPy * try using newer NumPy version for Windows Python 3.7 builds, to see if this helps `check_installed_package.py` * Add debug prints to _distributor_init.py * Debug print contents of .libs path for DLLs * Make sure libs_path exists before trying to debug print contents. * DEBUG: simplify CI matrix for iteration. * DEBUG: more debug prints near point of 32-bit Python 3.7 DLL resolution failure. * DEBUG: try moving into DLL dir * try to solve DLL resolution issues for older (32-bit) Python versions by temporarily moving into the path where they are stored prior to load attempts in `_distributor_init.py` * Revert "DEBUG: simplify CI matrix for iteration." This reverts commit 46e69e1d4c5d0261697cde178043dab1265ad62e. * MAINT: PR 70 cleanup * remove some debug prints * revert some debug changes * use `_distributor_init.py` with Python 3.6+ (all supported Python versions) * MAINT: PR 70 revisions * wrap the `_distributor_init.py` `os.chdir` code in a `try .. finally` block to ensure restoration of the working directory if something fails * substantially expand the comments describing the rationale for the working directory changes in `_distributor_init.py` * MAINT: PR 70 revisions * sync multibuild submodule to latest `devel` branch checkout in attempt to deal with Travis CI failures that just started appearing --- .travis.yml | 24 +++++++++++------------- _distributor_init.py | 41 +++++++++++++++++++++++++++++++++++++++-- appveyor.yml | 34 ++++++++++++---------------------- multibuild | 2 +- 4 files changed, 63 insertions(+), 38 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7888680..0d1a83c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,10 +4,10 @@ env: # Also see DAILY_COMMIT below - BUILD_COMMIT=master - PLAT=x86_64 - - NP_BUILD_DEP="numpy==1.13.3" + - NP_BUILD_DEP="numpy==1.14.5" - CYTHON_BUILD_DEP="Cython==0.29.14" - PYBIND11_BUILD_DEP="pybind11==2.4.3" - - NP_TEST_DEP="numpy==1.13.3" + - NP_TEST_DEP="numpy==1.14.5" - UNICODE_WIDTH=32 - MANYLINUX_URL="https://5cf40426d9f06eb7461d-6fe47d9331aba7cd62fc36c7196769e4.ssl.cf2.rackcdn.com" - WHEELHOUSE_UPLOADER_USERNAME=travis-worker @@ -19,13 +19,11 @@ env: - DAILY_COMMIT=master language: python -# Default Python version is usually 2.7 -python: 3.5 -sudo: required -dist: trusty +dist: bionic services: docker +os: linux -matrix: +jobs: exclude: # Exclude the default Python 3.5 build - python: 3.5 @@ -33,14 +31,14 @@ matrix: - os: linux env: - MB_PYTHON_VERSION=3.6 - - NP_BUILD_DEP=numpy==1.13.3 - - NP_TEST_DEP=numpy==1.13.3 + - NP_BUILD_DEP=numpy==1.14.5 + - NP_TEST_DEP=numpy==1.14.5 - os: linux env: - MB_PYTHON_VERSION=3.6 - PLAT=i686 - - NP_BUILD_DEP=numpy==1.13.3 - - NP_TEST_DEP=numpy==1.13.3 + - NP_BUILD_DEP=numpy==1.14.5 + - NP_TEST_DEP=numpy==1.14.5 - os: linux env: - MB_PYTHON_VERSION=3.7 @@ -71,8 +69,8 @@ matrix: language: generic env: - MB_PYTHON_VERSION=3.6 - - NP_BUILD_DEP=numpy==1.13.3 - - NP_TEST_DEP=numpy==1.13.3 + - NP_BUILD_DEP=numpy==1.14.5 + - NP_TEST_DEP=numpy==1.14.5 - os: osx language: generic env: diff --git a/_distributor_init.py b/_distributor_init.py index 8f6e1da..b339063 100644 --- a/_distributor_init.py +++ b/_distributor_init.py @@ -22,5 +22,42 @@ libs_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '.libs')) if os.path.isdir(libs_path): - for filename in glob.glob(os.path.join(libs_path, '*dll')): - WinDLL(os.path.abspath(filename)) + # for Python >= 3.8, DLL resolution ignores the PATH variable + # and the current working directory; see release notes: + # https://docs.python.org/3/whatsnew/3.8.html#bpo-36085-whatsnew + # Only the system paths, the directory containing the DLL, and + # directories added with add_dll_directory() are searched for + # load-time dependencies with Python >= 3.8 + + # this module was originally added to support DLL resolution in + # Python 3.8 because of the changes described above--providing the + # absolute paths to the DLLs allowed for proper resolution/loading + + # however, we also started to receive reports of problems with DLL + # resolution with Python 3.7 that were sometimes alleviated with + # inclusion of the _distributor_init.py module; see SciPy main + # repo gh-11826 + + # we noticed in scipy-wheels repo gh-70 that inclusion of + # _distributor_init.py in 32-bit wheels for Python 3.7 resulted + # in failures in DLL resolution (64-bit 3.7 did not) + # as a result, we decided to combine both the old (working directory) + # and new (absolute path to DLL location) DLL resolution mechanisms + # to improve the chances of resolving DLLs across a wider range of + # Python versions + + # we did not experiment with manipulating the PATH environment variable + # to include libs_path; it is not immediately clear if this would have + # robustness or security advantages over changing working directories + # as done below + + # we should remove the working directory shims when our minimum supported + # Python version is 3.8 and trust the improvements to secure DLL loading + # in the standard lib for Python >= 3.8 + try: + owd = os.getcwd() + os.chdir(libs_path) + for filename in glob.glob(os.path.join(libs_path, '*dll')): + WinDLL(os.path.abspath(filename)) + finally: + os.chdir(owd) diff --git a/appveyor.yml b/appveyor.yml index 9806d71..e6c8959 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,7 +17,7 @@ environment: OPENBLAS_32_SHA256: 06e3d38f01119afe5d6630d7ad310a873f8bede52fe71f2d0e2ebf3476194892 OPENBLAS_64_SHA256: 4d496081543c61bfb8069c1a12dfc2c0371cf9c59f9a4488e2e416dd4026357e CYTHON_BUILD_DEP: Cython==0.29.14 - NUMPY_TEST_DEP: numpy==1.13.3 + NUMPY_TEST_DEP: numpy==1.14.5 PYBIND11_BUILD_DEP: pybind11==2.4.3 TEST_MODE: fast APPVEYOR_SAVE_CACHE_ON_ERROR: true @@ -61,12 +61,12 @@ environment: - PYTHON: C:\Python36 PYTHON_VERSION: 3.6 PYTHON_ARCH: 32 - NUMPY_BUILD_DEP: numpy==1.13.3 + NUMPY_BUILD_DEP: numpy==1.14.5 - PYTHON: C:\Python36-x64 PYTHON_VERSION: 3.6 PYTHON_ARCH: 64 - NUMPY_BUILD_DEP: numpy==1.13.3 + NUMPY_BUILD_DEP: numpy==1.14.5 init: - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" @@ -154,7 +154,7 @@ install: # Replace numpy distutils with a version that can build with msvc + mingw-gfortran. - ps: | $PYTHON_VERSION = $env:PYTHON_VERSION - If ($PYTHON_VERSION -ne 3.8) { + If ([System.Version]"$PYTHON_VERSION" -lt [System.Version]"3.8") { $NumpyDir = $((python -c 'import os; import numpy; print(os.path.dirname(numpy.__file__))') | Out-String).Trim() rm -r -Force "$NumpyDir\distutils" mv numpy-distutils\numpy\distutils $NumpyDir @@ -166,17 +166,14 @@ build_script: - cd scipy - git checkout %BUILD_COMMIT% # we use a distribution file to assist in loading - # DLLs with recent Python versions (>= 3.8) + # DLLs - ps: | - $PYTHON_VERSION = $env:PYTHON_VERSION - If ($PYTHON_VERSION -eq 3.8) { - cd .. - $cwd = Get-Location - ls $cwd - rm -Force $cwd/scipy/scipy/_distributor_init.py - mv $cwd/_distributor_init.py $cwd/scipy/scipy/ - cd scipy - } + cd .. + $cwd = Get-Location + ls $cwd + rm -Force $cwd/scipy/scipy/_distributor_init.py + mv $cwd/_distributor_init.py $cwd/scipy/scipy/ + cd scipy # Append license text relevant for the built wheel - type ..\LICENSE_win32.txt >> LICENSE.txt # Copy over additional DLLs to bundle to the wheels. @@ -209,14 +206,7 @@ build_script: before_test: # Install test requirements. - - python -m pip install pytest pytest-xdist pytest-faulthandler pytest-env Pillow mpmath - - ps: | - $PYTHON_VERSION = $env:PYTHON_VERSION - If ($PYTHON_VERSION -eq 3.8) { - python -m pip install matplotlib==3.2.0rc1 - } Else { - python -m pip install matplotlib - } + - python -m pip install pytest pytest-xdist pytest-faulthandler pytest-env Pillow mpmath matplotlib # Upgrade numpy to the version used for testing - python -m pip install "%NUMPY_TEST_DEP%" diff --git a/multibuild b/multibuild index 0c4a992..2a3f785 160000 --- a/multibuild +++ b/multibuild @@ -1 +1 @@ -Subproject commit 0c4a9920fa4635419fcea61f10d6dc0cd50e3639 +Subproject commit 2a3f785096d791a5cd07c4320a041e37a055be5c