From 4c0fd39327577c26f3e7b47c2e2a4b6029902ba0 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Tue, 19 May 2020 21:52:52 +1200 Subject: [PATCH 01/33] Run Continuous Integration tests on Github Actions Signed-off-by: Wei Ji --- .github/workflows/ci_tests.yaml | 66 +++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/ci_tests.yaml diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml new file mode 100644 index 00000000000..e0cd6c9cc37 --- /dev/null +++ b/.github/workflows/ci_tests.yaml @@ -0,0 +1,66 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: PyGMT Tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + test: + name: ${{ matrix.os }} - Python ${{ matrix.python_version }} + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 3 + matrix: + python_version: [3.7] + os: [ubuntu-latest, macOS-latest, windows-latest] + env: + LD_LIBRARY_PATH: ${{ github.workspace }}/gmt/lib:$LD_LIBRARY_PATH + GMT_LIBRARY_PATH: ${{ github.workspace }}/gmt/lib + + steps: + # Checkout current git repository + - name: Checkout + uses: actions/checkout@v2.1.0 + + # Setup Miniconda + - name: Setup Miniconda + uses: goanpeca/setup-miniconda@v1 + with: + python-version: ${{ matrix.python-version }} + channels: conda-forge + + # Install GMT and other required dependencies from conda-forge + - name: Install GMT and required dependencies + shell: bash -l {0} + run: conda install gmt=6.0.0 numpy pandas xarray netCDF4 packaging + + # Show installed pkg information for postmortem diagnostic + - name: List installed packages + shell: bash -l {0} + run: conda list + + # Set environment variables to make conda GMT work for Windows + - name: Set GMT environment variables for Windows + run: | + echo ::set-env name=GMT_LIBRARY_PATH::%CONDA_PREFIX%\Library\bin + echo ::set-env name=GMT_SHAREDIR::%CONDA_PREFIX%\Library\share\gmt + if: startsWith(matrix.os, 'windows') + + # Install the package that we want to test + - name: Install the package + shell: bash -l {0} + run: | + python setup.py sdist --formats=zip + pip install dist/* + + # Run the tests + - name: Test with pytest + shell: bash -l {0} + run: | + conda install make + make test PYTEST_EXTRA="-r P" From 31340ee7101b50cc5c1e981c48f9f6df700a68a9 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Tue, 19 May 2020 22:11:41 +1200 Subject: [PATCH 02/33] Install make earlier Signed-off-by: Wei Ji --- .github/workflows/ci_tests.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index e0cd6c9cc37..b82a0a278a8 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -37,7 +37,7 @@ jobs: # Install GMT and other required dependencies from conda-forge - name: Install GMT and required dependencies shell: bash -l {0} - run: conda install gmt=6.0.0 numpy pandas xarray netCDF4 packaging + run: conda install gmt=6.0.0 numpy pandas xarray netCDF4 packaging make # Show installed pkg information for postmortem diagnostic - name: List installed packages @@ -61,6 +61,4 @@ jobs: # Run the tests - name: Test with pytest shell: bash -l {0} - run: | - conda install make - make test PYTEST_EXTRA="-r P" + run: make test PYTEST_EXTRA="-r P" From 9a4e7bc5fa0adb8982323fc446b2e74b6a980220 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Wed, 20 May 2020 18:30:10 +1200 Subject: [PATCH 03/33] Set Linux/MacOS environment variables properly Signed-off-by: Wei Ji --- .github/workflows/ci_tests.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index b82a0a278a8..b289eb1e542 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -18,9 +18,6 @@ jobs: matrix: python_version: [3.7] os: [ubuntu-latest, macOS-latest, windows-latest] - env: - LD_LIBRARY_PATH: ${{ github.workspace }}/gmt/lib:$LD_LIBRARY_PATH - GMT_LIBRARY_PATH: ${{ github.workspace }}/gmt/lib steps: # Checkout current git repository @@ -44,12 +41,19 @@ jobs: shell: bash -l {0} run: conda list - # Set environment variables to make conda GMT work for Windows + # Set environment variables for Linux/MacOS + - name: Set GMT environment variables for Linux/MacOS + run: | + echo ::set-env name=LD_LIBRARY_PATH/gmt/lib:$LD_LIBRARY_PATH + echo ::set-env name=GMT_LIBRARY_PATH/gmt/lib + if: matrix.os != 'windows-latest' + + # Set environment variables for Windows - name: Set GMT environment variables for Windows run: | echo ::set-env name=GMT_LIBRARY_PATH::%CONDA_PREFIX%\Library\bin echo ::set-env name=GMT_SHAREDIR::%CONDA_PREFIX%\Library\share\gmt - if: startsWith(matrix.os, 'windows') + if: matrix.os == 'windows-latest' # Install the package that we want to test - name: Install the package From 51472dc7eead27a2977ac1f481432f8f6aa6c75a Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Wed, 20 May 2020 18:31:14 +1200 Subject: [PATCH 04/33] Install pytest and other dev dependencies Signed-off-by: Wei Ji --- .github/workflows/ci_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index b289eb1e542..cf59ede02e4 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -34,7 +34,7 @@ jobs: # Install GMT and other required dependencies from conda-forge - name: Install GMT and required dependencies shell: bash -l {0} - run: conda install gmt=6.0.0 numpy pandas xarray netCDF4 packaging make + run: conda install gmt=6.0.0 numpy pandas xarray netCDF4 packaging make pytest pytest-cov pytest-mpl sphinx jinja2 docutils ipython # Show installed pkg information for postmortem diagnostic - name: List installed packages From 9c2e393dfcc80c6816ae80572b71eba1230ef3da Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Wed, 20 May 2020 18:32:01 +1200 Subject: [PATCH 05/33] Don't fail-fast and cancel all jobs See https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast Signed-off-by: Wei Ji --- .github/workflows/ci_tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index cf59ede02e4..e9591b7a445 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -15,6 +15,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: max-parallel: 3 + fail-fast: false matrix: python_version: [3.7] os: [ubuntu-latest, macOS-latest, windows-latest] From 2220702539f03e4b54c075346e616fec27a8737b Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Wed, 20 May 2020 19:14:47 +1200 Subject: [PATCH 06/33] Update setup-miniconda Github Action from v1 to v1.3.1 Signed-off-by: Wei Ji --- .github/workflows/ci_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index e9591b7a445..97b05d6ac44 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -27,7 +27,7 @@ jobs: # Setup Miniconda - name: Setup Miniconda - uses: goanpeca/setup-miniconda@v1 + uses: goanpeca/setup-miniconda@v1.3.1 with: python-version: ${{ matrix.python-version }} channels: conda-forge From 6936b3ced635b10254550448edbf85ae7692decf Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Thu, 21 May 2020 18:57:09 +1200 Subject: [PATCH 07/33] Don't set any environment variables for all OSes --- .github/workflows/ci_tests.yaml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 97b05d6ac44..974a0078709 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -42,20 +42,6 @@ jobs: shell: bash -l {0} run: conda list - # Set environment variables for Linux/MacOS - - name: Set GMT environment variables for Linux/MacOS - run: | - echo ::set-env name=LD_LIBRARY_PATH/gmt/lib:$LD_LIBRARY_PATH - echo ::set-env name=GMT_LIBRARY_PATH/gmt/lib - if: matrix.os != 'windows-latest' - - # Set environment variables for Windows - - name: Set GMT environment variables for Windows - run: | - echo ::set-env name=GMT_LIBRARY_PATH::%CONDA_PREFIX%\Library\bin - echo ::set-env name=GMT_SHAREDIR::%CONDA_PREFIX%\Library\share\gmt - if: matrix.os == 'windows-latest' - # Install the package that we want to test - name: Install the package shell: bash -l {0} From 6906426d92e13443685b0d909557eca8d340800e Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Fri, 5 Jun 2020 15:37:46 +1200 Subject: [PATCH 08/33] Cache GMT tutorial files in ${{ github.workspace }}/.gmt on 20200605 Try just caching ~/.gmt/cache and ~/.gmt/server. --- .github/workflows/ci_tests.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 974a0078709..4e77fe51bc0 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -42,6 +42,23 @@ jobs: shell: bash -l {0} run: conda list + # Cache the ${HOME}/.gmt directory, for docs and testing + - name: Cache GMT directory + uses: actions/cache@v2 + id: cache + with: + path: | + ${{ github.workspace }}/.gmt/cache + ${{ github.workspace }}/.gmt/server + key: cache-gmt-${{ github.ref }}-20200605-2 + restore-keys: cache-gmt-refs/heads/master- + + # Download remote files, if not already cached + - name: Download remote data + shell: bash -l {0} + run: gmt which -Gu @earth_relief_10m @earth_relief_60m @ridge.txt @Table_5_11.txt @tut_bathy.nc @tut_quakes.ngdc @tut_ship.xyz + if: steps.cache.outputs.cache-hit != 'true' + # Install the package that we want to test - name: Install the package shell: bash -l {0} From d2ce09b187426a5c81f47fc0e935064503d2b712 Mon Sep 17 00:00:00 2001 From: Wei Ji <23487320+weiji14@users.noreply.github.com> Date: Mon, 8 Jun 2020 09:41:31 +1200 Subject: [PATCH 09/33] Fix typo python_version to python-version Co-authored-by: Dongdong Tian --- .github/workflows/ci_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 4e77fe51bc0..91399131ced 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -17,7 +17,7 @@ jobs: max-parallel: 3 fail-fast: false matrix: - python_version: [3.7] + python-version: [3.7] os: [ubuntu-latest, macOS-latest, windows-latest] steps: From e40708161d0b009d151847fc8f23a6552fd98ea9 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Tue, 9 Jun 2020 00:49:31 -0400 Subject: [PATCH 10/33] Fix python_version to python-version --- .github/workflows/ci_tests.yaml | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 91399131ced..5e8a6defc3e 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -11,7 +11,7 @@ on: jobs: test: - name: ${{ matrix.os }} - Python ${{ matrix.python_version }} + name: ${{ matrix.os }} - Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: max-parallel: 3 @@ -48,16 +48,30 @@ jobs: id: cache with: path: | - ${{ github.workspace }}/.gmt/cache - ${{ github.workspace }}/.gmt/server - key: cache-gmt-${{ github.ref }}-20200605-2 + ~/.gmt/cache + ~/.gmt/server + key: cache-gmt-${{ runner.os }}-20200609 restore-keys: cache-gmt-refs/heads/master- + - name: Download remote data using wget (Linux & Windows) + shell: bash -l {0} + run: | + if [ "$RUNNER_OS" == "Windows" ]; then choco install wget; fi # install wget on Windows + mkdir ~/.gmt ~/.gmt/cache ~/.gmt/server + wget --no-check-certificate https://oceania.generic-mapping-tools.org/gmt_hash_server.txt -P ~/.gmt/server/ + for data in earth_relief_60m.grd earth_relief_30m.grd earth_relief_10m.grd; do + wget --no-check-certificate https://oceania.generic-mapping-tools.org/${data} -P ~/.gmt/server/ + done + for data in ridge.txt Table_5_11.txt tut_bathy.nc tut_quakes.ngdc tut_ship.xyz usgs_quakes_22.txt; do + wget --no-check-certificate https://oceania.generic-mapping-tools.org/cache/${data} -P ~/.gmt/cache/ + done + if: steps.cache.outputs.cache-hit != 'true' && runner.os != 'macOS' + # Download remote files, if not already cached - - name: Download remote data + - name: Download remote data (macOS) shell: bash -l {0} - run: gmt which -Gu @earth_relief_10m @earth_relief_60m @ridge.txt @Table_5_11.txt @tut_bathy.nc @tut_quakes.ngdc @tut_ship.xyz - if: steps.cache.outputs.cache-hit != 'true' + run: gmt which -Gu @earth_relief_60m @earth_relief_30m @earth_relief_10m @ridge.txt @Table_5_11.txt @tut_bathy.nc @tut_quakes.ngdc @tut_ship.xyz @usgs_quakes_22.txt + if: steps.cache.outputs.cache-hit != 'true' && runner.os == 'macOS' # Install the package that we want to test - name: Install the package From 841f937aa60eddb718178ec06748c405bc691fd9 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Wed, 10 Jun 2020 01:53:40 -0400 Subject: [PATCH 11/33] Disable tests to enable caches --- .github/workflows/ci_tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 5e8a6defc3e..05b1889a3b7 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -81,6 +81,6 @@ jobs: pip install dist/* # Run the tests - - name: Test with pytest - shell: bash -l {0} - run: make test PYTEST_EXTRA="-r P" + #- name: Test with pytest + # shell: bash -l {0} + # run: make test PYTEST_EXTRA="-r P" From 9fd73f0a6439cd1fdba4ddb650549bda59328a99 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Wed, 10 Jun 2020 02:50:28 -0400 Subject: [PATCH 12/33] Re-enable tests to check caches --- .github/workflows/ci_tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 05b1889a3b7..5e8a6defc3e 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -81,6 +81,6 @@ jobs: pip install dist/* # Run the tests - #- name: Test with pytest - # shell: bash -l {0} - # run: make test PYTEST_EXTRA="-r P" + - name: Test with pytest + shell: bash -l {0} + run: make test PYTEST_EXTRA="-r P" From 1a23b55fc39c56e5f2fb28802a4267961a9e7b93 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Wed, 10 Jun 2020 11:12:06 -0400 Subject: [PATCH 13/33] Change github action name --- .github/workflows/ci_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 5e8a6defc3e..ea0a57c23a2 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: PyGMT Tests +name: Tests on: push: From 0ba3d7e42993c3598ccf0142331ea190aec64513 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Wed, 10 Jun 2020 11:12:36 -0400 Subject: [PATCH 14/33] Simplify job name --- .github/workflows/ci_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index ea0a57c23a2..837536f4c31 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -11,7 +11,7 @@ on: jobs: test: - name: ${{ matrix.os }} - Python ${{ matrix.python-version }} + name: ${{ runner.os }} - Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: max-parallel: 3 From c744db9670b1649b2046bcc1c218a2c19d2b8f13 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Wed, 10 Jun 2020 11:15:10 -0400 Subject: [PATCH 15/33] The versioneer pacakge need tags to determine the PyGMT version --- .github/workflows/ci_tests.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 837536f4c31..9a83a361fdc 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -25,6 +25,10 @@ jobs: - name: Checkout uses: actions/checkout@v2.1.0 + # See https://github.com/warner/python-versioneer/issues/206 + - name: Fetch all history for all tags and branches + run: git fetch --prune --unshallow + # Setup Miniconda - name: Setup Miniconda uses: goanpeca/setup-miniconda@v1.3.1 From 3a06afca6b53bae0b7e90ecf46e66b121e366b36 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Wed, 10 Jun 2020 11:43:58 -0400 Subject: [PATCH 16/33] Change runner.os to matrix.os --- .github/workflows/ci_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 9a83a361fdc..999d6dab2fc 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -11,7 +11,7 @@ on: jobs: test: - name: ${{ runner.os }} - Python ${{ matrix.python-version }} + name: ${{ matrix.os }} - Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: max-parallel: 3 From 05c225f380be5a8e545c7945baa7190563ac2d38 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Wed, 10 Jun 2020 11:58:39 -0400 Subject: [PATCH 17/33] Improve the docs --- .github/workflows/ci_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 999d6dab2fc..e37af9b1ee6 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -14,7 +14,6 @@ jobs: name: ${{ matrix.os }} - Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: - max-parallel: 3 fail-fast: false matrix: python-version: [3.7] @@ -25,7 +24,7 @@ jobs: - name: Checkout uses: actions/checkout@v2.1.0 - # See https://github.com/warner/python-versioneer/issues/206 + # Workaround for versioneer, see https://github.com/warner/python-versioneer/issues/206 - name: Fetch all history for all tags and branches run: git fetch --prune --unshallow @@ -57,6 +56,7 @@ jobs: key: cache-gmt-${{ runner.os }}-20200609 restore-keys: cache-gmt-refs/heads/master- + # Workaround for the timeouts of 'gmt which' on Linux and Windows - name: Download remote data using wget (Linux & Windows) shell: bash -l {0} run: | From 8bd8a6663a2394df624f41df1dd29919cf5d99d3 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Wed, 10 Jun 2020 12:02:49 -0400 Subject: [PATCH 18/33] Add a cron job for daily tests --- .github/workflows/ci_tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index e37af9b1ee6..5bf72145445 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -8,6 +8,9 @@ on: branches: [ master ] pull_request: branches: [ master ] + # Schedule daily tests + schedule: + - cron: '0 0 * * *' jobs: test: From c12cfffa50815836daedd60fc4c786c888a2c207 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Sun, 28 Jun 2020 22:58:42 -0400 Subject: [PATCH 19/33] Change earth_relief_60m to earth_relief_01d --- .github/workflows/ci_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 5bf72145445..75e255106a1 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -66,7 +66,7 @@ jobs: if [ "$RUNNER_OS" == "Windows" ]; then choco install wget; fi # install wget on Windows mkdir ~/.gmt ~/.gmt/cache ~/.gmt/server wget --no-check-certificate https://oceania.generic-mapping-tools.org/gmt_hash_server.txt -P ~/.gmt/server/ - for data in earth_relief_60m.grd earth_relief_30m.grd earth_relief_10m.grd; do + for data in earth_relief_01d.grd earth_relief_30m.grd earth_relief_10m.grd; do wget --no-check-certificate https://oceania.generic-mapping-tools.org/${data} -P ~/.gmt/server/ done for data in ridge.txt Table_5_11.txt tut_bathy.nc tut_quakes.ngdc tut_ship.xyz usgs_quakes_22.txt; do @@ -77,7 +77,7 @@ jobs: # Download remote files, if not already cached - name: Download remote data (macOS) shell: bash -l {0} - run: gmt which -Gu @earth_relief_60m @earth_relief_30m @earth_relief_10m @ridge.txt @Table_5_11.txt @tut_bathy.nc @tut_quakes.ngdc @tut_ship.xyz @usgs_quakes_22.txt + run: gmt which -Gu @earth_relief_01d @earth_relief_30m @earth_relief_10m @ridge.txt @Table_5_11.txt @tut_bathy.nc @tut_quakes.ngdc @tut_ship.xyz @usgs_quakes_22.txt if: steps.cache.outputs.cache-hit != 'true' && runner.os == 'macOS' # Install the package that we want to test From 9179bc265880f079882a15ea1dbe8e2247180d8a Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Sun, 28 Jun 2020 23:31:17 -0400 Subject: [PATCH 20/33] Let conda install packages listed in requirements.txt --- .github/workflows/ci_tests.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 75e255106a1..99f036a3f04 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -41,7 +41,10 @@ jobs: # Install GMT and other required dependencies from conda-forge - name: Install GMT and required dependencies shell: bash -l {0} - run: conda install gmt=6.0.0 numpy pandas xarray netCDF4 packaging make pytest pytest-cov pytest-mpl sphinx jinja2 docutils ipython + run: | + conda install --yes --file requirements.txt + conda install --yes --file requirements-dev.txt + conda install gmt=6.0.0 make # Show installed pkg information for postmortem diagnostic - name: List installed packages From 8662be01c35f64c9309c2a23356f3c6dd4552a60 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Mon, 29 Jun 2020 00:40:56 -0400 Subject: [PATCH 21/33] Single conda command to install all packages --- .github/workflows/ci_tests.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 99f036a3f04..c516b1a800f 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -42,9 +42,13 @@ jobs: - name: Install GMT and required dependencies shell: bash -l {0} run: | - conda install --yes --file requirements.txt - conda install --yes --file requirements-dev.txt - conda install gmt=6.0.0 make + requirements_file=full-conda-requirements.txt + cat requirements.txt requirements-dev.txt > $requirements_file + cat << EOF >> $requirements_file + gmt=6.0.0 + make + EOF + conda install --yes --file $requirements_file # Show installed pkg information for postmortem diagnostic - name: List installed packages From b5caae6f51235cde45bc267339730884f7da1bce Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Mon, 22 Jun 2020 23:57:49 +1200 Subject: [PATCH 22/33] Attempt to test GMT master on Github Actions Trying out Ubuntu only for now. --- .github/workflows/ci_tests.yaml | 81 +++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index c516b1a800f..124238d5b66 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -98,3 +98,84 @@ jobs: - name: Test with pytest shell: bash -l {0} run: make test PYTEST_EXTRA="-r P" + + test_gmt_master: + name: GMT master - ${{ matrix.os }} - Python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [3.7] + os: [ubuntu-latest] + env: + # LD_LIBRARY_PATH: ${{ github.workspace }}/gmt/lib:$LD_LIBRARY_PATH + GMT_LIBRARY_PATH: ${{ github.workspace }}/gmt/lib + + steps: + # Checkout current git repository + - name: Checkout + uses: actions/checkout@v2.1.0 + + # Workaround for versioneer, see https://github.com/warner/python-versioneer/issues/206 + - name: Fetch all history for all tags and branches + run: git fetch --prune --unshallow + + # Setup Miniconda + - name: Setup Miniconda + uses: goanpeca/setup-miniconda@v1.3.1 + with: + python-version: ${{ matrix.python-version }} + channels: conda-forge + + # Install GMT master branch + - name: Install GMT from master + shell: bash -l {0} + run: | + echo "::add-path::$GITHUB_WORKSPACE/gmt/bin" + apt -qq update + DEBIAN_FRONTEND=noninteractive apt install -y cmake xdg-utils libcurl4-openssl-dev + curl https://raw.githubusercontent.com/GenericMappingTools/gmt/master/ci/build-gmt-master.sh | bash + + # Cache the ${HOME}/.gmt directory, for docs and testing + - name: Cache GMT directory + uses: actions/cache@v2 + id: cache2 + with: + path: | + ~/.gmt/cache + ~/.gmt/server + key: cache2-gmt-${{ runner.os }}-20200609 + restore-keys: cache2-gmt-refs/heads/master- + + # Workaround for the timeouts of 'gmt which' on Linux and Windows + - name: Download remote data using wget (Linux & Windows) + shell: bash -l {0} + run: | + if [ "$RUNNER_OS" == "Windows" ]; then choco install wget; fi # install wget on Windows + mkdir ~/.gmt ~/.gmt/cache ~/.gmt/server + wget --no-check-certificate https://oceania.generic-mapping-tools.org/gmt_hash_server.txt -P ~/.gmt/server/ + for data in earth_relief_60m.grd earth_relief_30m.grd earth_relief_10m.grd; do + wget --no-check-certificate https://oceania.generic-mapping-tools.org/${data} -P ~/.gmt/server/ + done + for data in ridge.txt Table_5_11.txt tut_bathy.nc tut_quakes.ngdc tut_ship.xyz usgs_quakes_22.txt; do + wget --no-check-certificate https://oceania.generic-mapping-tools.org/cache/${data} -P ~/.gmt/cache/ + done + if: steps.cache2.outputs.cache-hit != 'true' && runner.os != 'macOS' + + # Download remote files, if not already cached + - name: Download remote data (macOS) + shell: bash -l {0} + run: gmt which -Gu @earth_relief_60m @earth_relief_30m @earth_relief_10m @ridge.txt @Table_5_11.txt @tut_bathy.nc @tut_quakes.ngdc @tut_ship.xyz @usgs_quakes_22.txt + if: steps.cache2.outputs.cache-hit != 'true' && runner.os == 'macOS' + + # Install the package that we want to test + - name: Install the package + shell: bash -l {0} + run: | + python setup.py sdist --formats=zip + pip install dist/* + + # Run the tests + - name: Test with pytest + shell: bash -l {0} + run: make test PYTEST_EXTRA="-r P" From 127c8518525df0a1c1825ebca230e0f3e65bcef7 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Mon, 29 Jun 2020 16:39:12 +1200 Subject: [PATCH 23/33] Use conda to install build dependencies instead of apt Make it more cross platform. --- .github/workflows/ci_tests.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 124238d5b66..fa5bedc1ca8 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -127,14 +127,15 @@ jobs: python-version: ${{ matrix.python-version }} channels: conda-forge + # Install build dependencies from conda-forge + - name: Install build dependencies + shell: bash -l {0} + run: conda install cmake libblas libcblas liblapack fftw gdal ghostscript libnetcdf hdf5 zlib curl pcre gshhg-gmt dcw-gmt + # Install GMT master branch - name: Install GMT from master shell: bash -l {0} - run: | - echo "::add-path::$GITHUB_WORKSPACE/gmt/bin" - apt -qq update - DEBIAN_FRONTEND=noninteractive apt install -y cmake xdg-utils libcurl4-openssl-dev - curl https://raw.githubusercontent.com/GenericMappingTools/gmt/master/ci/build-gmt-master.sh | bash + run: curl https://raw.githubusercontent.com/GenericMappingTools/gmt/master/ci/build-gmt-master.sh | bash # Cache the ${HOME}/.gmt directory, for docs and testing - name: Cache GMT directory @@ -144,7 +145,7 @@ jobs: path: | ~/.gmt/cache ~/.gmt/server - key: cache2-gmt-${{ runner.os }}-20200609 + key: cache2-gmt-${{ runner.os }}-20200629 restore-keys: cache2-gmt-refs/heads/master- # Workaround for the timeouts of 'gmt which' on Linux and Windows From ba7b6ca2a4fde7e7a521ddd3845f2d67873a8ee1 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Mon, 29 Jun 2020 18:55:05 +1200 Subject: [PATCH 24/33] Set GMT_LIBRARY_PATH to gmt-install-dir/lib Also explicitly install it to the github.workspace directory instead of $HOME, and set the proper cache key. Co-Authored-By: Dongdong Tian --- .github/workflows/ci_tests.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index fa5bedc1ca8..d692a359aaa 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -109,7 +109,8 @@ jobs: os: [ubuntu-latest] env: # LD_LIBRARY_PATH: ${{ github.workspace }}/gmt/lib:$LD_LIBRARY_PATH - GMT_LIBRARY_PATH: ${{ github.workspace }}/gmt/lib + GMT_INSTALL_DIR: ${{ github.workspace }}/gmt-install-dir + GMT_LIBRARY_PATH: ${{ github.workspace }}/gmt-install-dir/lib steps: # Checkout current git repository @@ -145,7 +146,7 @@ jobs: path: | ~/.gmt/cache ~/.gmt/server - key: cache2-gmt-${{ runner.os }}-20200629 + key: cache2-gmt-${{ github.ref }}-${{ runner.os }}-20200629 restore-keys: cache2-gmt-refs/heads/master- # Workaround for the timeouts of 'gmt which' on Linux and Windows From b87de5d217c93d87c4979476dc9e17a6d6470090 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Mon, 29 Jun 2020 19:30:04 +1200 Subject: [PATCH 25/33] Install pytest et al --- .github/workflows/ci_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index d692a359aaa..946da1dfa29 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -131,7 +131,7 @@ jobs: # Install build dependencies from conda-forge - name: Install build dependencies shell: bash -l {0} - run: conda install cmake libblas libcblas liblapack fftw gdal ghostscript libnetcdf hdf5 zlib curl pcre gshhg-gmt dcw-gmt + run: conda install cmake libblas libcblas liblapack fftw gdal ghostscript libnetcdf hdf5 zlib curl pcre gshhg-gmt dcw-gmt pytest pytest-cov pytest-mpl # Install GMT master branch - name: Install GMT from master From 2dc2e6e3cd54d92f9bffb0e6658be5180917a47d Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Mon, 29 Jun 2020 19:56:59 +1200 Subject: [PATCH 26/33] Test on Ubuntu/macOS/Windows Also just use a default bash shell for every step. --- .github/workflows/ci_tests.yaml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 946da1dfa29..75a180e86c8 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -106,11 +106,14 @@ jobs: fail-fast: false matrix: python-version: [3.7] - os: [ubuntu-latest] + os: [ubuntu-20.04, macOS-10.15, windows-2019] env: # LD_LIBRARY_PATH: ${{ github.workspace }}/gmt/lib:$LD_LIBRARY_PATH GMT_INSTALL_DIR: ${{ github.workspace }}/gmt-install-dir GMT_LIBRARY_PATH: ${{ github.workspace }}/gmt-install-dir/lib + defaults: + run: + shell: bash -l {0} steps: # Checkout current git repository @@ -130,12 +133,10 @@ jobs: # Install build dependencies from conda-forge - name: Install build dependencies - shell: bash -l {0} run: conda install cmake libblas libcblas liblapack fftw gdal ghostscript libnetcdf hdf5 zlib curl pcre gshhg-gmt dcw-gmt pytest pytest-cov pytest-mpl # Install GMT master branch - name: Install GMT from master - shell: bash -l {0} run: curl https://raw.githubusercontent.com/GenericMappingTools/gmt/master/ci/build-gmt-master.sh | bash # Cache the ${HOME}/.gmt directory, for docs and testing @@ -151,12 +152,11 @@ jobs: # Workaround for the timeouts of 'gmt which' on Linux and Windows - name: Download remote data using wget (Linux & Windows) - shell: bash -l {0} run: | if [ "$RUNNER_OS" == "Windows" ]; then choco install wget; fi # install wget on Windows mkdir ~/.gmt ~/.gmt/cache ~/.gmt/server wget --no-check-certificate https://oceania.generic-mapping-tools.org/gmt_hash_server.txt -P ~/.gmt/server/ - for data in earth_relief_60m.grd earth_relief_30m.grd earth_relief_10m.grd; do + for data in earth_relief_01d.grd earth_relief_30m.grd earth_relief_10m.grd; do wget --no-check-certificate https://oceania.generic-mapping-tools.org/${data} -P ~/.gmt/server/ done for data in ridge.txt Table_5_11.txt tut_bathy.nc tut_quakes.ngdc tut_ship.xyz usgs_quakes_22.txt; do @@ -166,18 +166,15 @@ jobs: # Download remote files, if not already cached - name: Download remote data (macOS) - shell: bash -l {0} run: gmt which -Gu @earth_relief_60m @earth_relief_30m @earth_relief_10m @ridge.txt @Table_5_11.txt @tut_bathy.nc @tut_quakes.ngdc @tut_ship.xyz @usgs_quakes_22.txt if: steps.cache2.outputs.cache-hit != 'true' && runner.os == 'macOS' # Install the package that we want to test - name: Install the package - shell: bash -l {0} run: | python setup.py sdist --formats=zip pip install dist/* # Run the tests - name: Test with pytest - shell: bash -l {0} run: make test PYTEST_EXTRA="-r P" From 928ab48c8d2de9e287184a7ad630b822124b5d46 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Mon, 29 Jun 2020 20:21:49 +1200 Subject: [PATCH 27/33] Link to gmt binary on macOS and skip Windows for now --- .github/workflows/ci_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 75a180e86c8..0ead43777e7 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -106,7 +106,7 @@ jobs: fail-fast: false matrix: python-version: [3.7] - os: [ubuntu-20.04, macOS-10.15, windows-2019] + os: [ubuntu-20.04, macOS-10.15] env: # LD_LIBRARY_PATH: ${{ github.workspace }}/gmt/lib:$LD_LIBRARY_PATH GMT_INSTALL_DIR: ${{ github.workspace }}/gmt-install-dir @@ -166,7 +166,7 @@ jobs: # Download remote files, if not already cached - name: Download remote data (macOS) - run: gmt which -Gu @earth_relief_60m @earth_relief_30m @earth_relief_10m @ridge.txt @Table_5_11.txt @tut_bathy.nc @tut_quakes.ngdc @tut_ship.xyz @usgs_quakes_22.txt + run: $GMT_INSTALL_DIR/bin/gmt which -Gu @earth_relief_60m @earth_relief_30m @earth_relief_10m @ridge.txt @Table_5_11.txt @tut_bathy.nc @tut_quakes.ngdc @tut_ship.xyz @usgs_quakes_22.txt if: steps.cache2.outputs.cache-hit != 'true' && runner.os == 'macOS' # Install the package that we want to test From 575baa7a680adc0a40a97bfb5c80f5b94c9b427b Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Sun, 12 Jul 2020 22:28:40 +1200 Subject: [PATCH 28/33] Update cache key to 20200712 and use GMT 6.1.0 specific dataset names --- .github/workflows/ci_tests.yaml | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index f97b25505a3..e3ddf1dbbf0 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -137,15 +137,14 @@ jobs: steps: # Checkout current git repository - name: Checkout - uses: actions/checkout@v2.1.0 - - # Workaround for versioneer, see https://github.com/warner/python-versioneer/issues/206 - - name: Fetch all history for all tags and branches - run: git fetch --prune --unshallow + uses: actions/checkout@v2.3.1 + with: + # fecth all history so that versioneer works + fetch-depth: 0 # Setup Miniconda - name: Setup Miniconda - uses: goanpeca/setup-miniconda@v1.3.1 + uses: goanpeca/setup-miniconda@v1.6.0 with: python-version: ${{ matrix.python-version }} channels: conda-forge @@ -166,26 +165,28 @@ jobs: path: | ~/.gmt/cache ~/.gmt/server - key: cache2-gmt-${{ github.ref }}-${{ runner.os }}-20200629 - restore-keys: cache2-gmt-refs/heads/master- + key: cache2-gmt-${{ runner.os }}-${{ github.ref }}-20200711 + restore-keys: cache2-gmt-${{ runner.os }}-refs/heads/master- # Workaround for the timeouts of 'gmt which' on Linux and Windows - name: Download remote data using wget (Linux & Windows) run: | if [ "$RUNNER_OS" == "Windows" ]; then choco install wget; fi # install wget on Windows - mkdir ~/.gmt ~/.gmt/cache ~/.gmt/server + mkdir -p ~/.gmt ~/.gmt/cache ~/.gmt/server/earth/earth_relief/ wget --no-check-certificate https://oceania.generic-mapping-tools.org/gmt_hash_server.txt -P ~/.gmt/server/ - for data in earth_relief_01d.grd earth_relief_30m.grd earth_relief_10m.grd; do - wget --no-check-certificate https://oceania.generic-mapping-tools.org/${data} -P ~/.gmt/server/ + for data in earth_relief_01d_p.grd earth_relief_01d_g.grd earth_relief_30m_p.grd earth_relief_30m_g.grd earth_relief_10m_p.grd earth_relief_10m_g.grd; do + wget --no-check-certificate https://oceania.generic-mapping-tools.org/server/earth/earth_relief/${data} -P ~/.gmt/server/earth/earth_relief/ done - for data in ridge.txt Table_5_11.txt tut_bathy.nc tut_quakes.ngdc tut_ship.xyz usgs_quakes_22.txt; do + for data in ridge.txt Table_5_11.txt test.dat.nc tut_bathy.nc tut_quakes.ngdc tut_ship.xyz usgs_quakes_22.txt; do wget --no-check-certificate https://oceania.generic-mapping-tools.org/cache/${data} -P ~/.gmt/cache/ done if: steps.cache2.outputs.cache-hit != 'true' && runner.os != 'macOS' # Download remote files, if not already cached - name: Download remote data (macOS) - run: $GMT_INSTALL_DIR/bin/gmt which -Gu @earth_relief_60m @earth_relief_30m @earth_relief_10m @ridge.txt @Table_5_11.txt @tut_bathy.nc @tut_quakes.ngdc @tut_ship.xyz @usgs_quakes_22.txt + run: | + $GMT_INSTALL_DIR/bin/gmt which -Ga @earth_relief_10m_p @earth_relief_10m_g @earth_relief_30m_p @earth_relief_30m_g @earth_relief_01d_p @earth_relief_01d_g + $GMT_INSTALL_DIR/bin/gmt which -Ga @ridge.txt @Table_5_11.txt @test.dat.nc @tut_bathy.nc @tut_quakes.ngdc @tut_ship.xyz @usgs_quakes_22.txt if: steps.cache2.outputs.cache-hit != 'true' && runner.os == 'macOS' # Install the package that we want to test From a4a6d7a7ba30c68e289aa280cce8d1a92c739d20 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Sun, 12 Jul 2020 22:55:08 +1200 Subject: [PATCH 29/33] Add ipython for test_figure_show --- .github/workflows/ci_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index e3ddf1dbbf0..2e0c9e24374 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -151,7 +151,7 @@ jobs: # Install build dependencies from conda-forge - name: Install build dependencies - run: conda install cmake libblas libcblas liblapack fftw gdal ghostscript libnetcdf hdf5 zlib curl pcre gshhg-gmt dcw-gmt pytest pytest-cov pytest-mpl + run: conda install cmake libblas libcblas liblapack fftw gdal ghostscript libnetcdf hdf5 zlib curl pcre gshhg-gmt dcw-gmt ipython pytest pytest-cov pytest-mpl # Install GMT master branch - name: Install GMT from master From 65ec7ce3a96ad2faa6efac26b2beacac6ce9c13c Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Thu, 16 Jul 2020 20:45:21 +1200 Subject: [PATCH 30/33] Use remote file cache artifacts from Github --- .github/workflows/ci_tests.yaml | 42 ++++++++++----------------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 829fd770777..73d63170f40 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -163,37 +163,21 @@ jobs: - name: Install GMT from master run: curl https://raw.githubusercontent.com/GenericMappingTools/gmt/master/ci/build-gmt-master.sh | bash - # Cache the ${HOME}/.gmt directory, for docs and testing - - name: Cache GMT directory - uses: actions/cache@v2 - id: cache2 + # Download cached remote files (artifacts) from Github + - name: Download remote data from Github + uses: dawidd6/action-download-artifact@v2.6.3 with: - path: | - ~/.gmt/cache - ~/.gmt/server - key: cache2-gmt-${{ runner.os }}-${{ github.ref }}-20200711 - restore-keys: cache2-gmt-${{ runner.os }}-refs/heads/master- - - # Workaround for the timeouts of 'gmt which' on Linux and Windows - - name: Download remote data using wget (Linux & Windows) - run: | - if [ "$RUNNER_OS" == "Windows" ]; then choco install wget; fi # install wget on Windows - mkdir -p ~/.gmt ~/.gmt/cache ~/.gmt/server/earth/earth_relief/ - wget --no-check-certificate https://oceania.generic-mapping-tools.org/gmt_hash_server.txt -P ~/.gmt/server/ - for data in earth_relief_01d_p.grd earth_relief_01d_g.grd earth_relief_30m_p.grd earth_relief_30m_g.grd earth_relief_10m_p.grd earth_relief_10m_g.grd; do - wget --no-check-certificate https://oceania.generic-mapping-tools.org/server/earth/earth_relief/${data} -P ~/.gmt/server/earth/earth_relief/ - done - for data in ridge.txt Table_5_11.txt test.dat.nc tut_bathy.nc tut_quakes.ngdc tut_ship.xyz usgs_quakes_22.txt; do - wget --no-check-certificate https://oceania.generic-mapping-tools.org/cache/${data} -P ~/.gmt/cache/ - done - if: steps.cache2.outputs.cache-hit != 'true' && runner.os != 'macOS' - - # Download remote files, if not already cached - - name: Download remote data (macOS) + workflow: cache_data.yaml + name: gmt-cache + path: .gmt + + # Move downloaded files to ~/.gmt directory and list them + - name: Move and list downloaded remote files + shell: bash -l {0} run: | - $GMT_INSTALL_DIR/bin/gmt which -Ga @earth_relief_10m_p @earth_relief_10m_g @earth_relief_30m_p @earth_relief_30m_g @earth_relief_01d_p @earth_relief_01d_g - $GMT_INSTALL_DIR/bin/gmt which -Ga @ridge.txt @Table_5_11.txt @test.dat.nc @tut_bathy.nc @tut_quakes.ngdc @tut_ship.xyz @usgs_quakes_22.txt - if: steps.cache2.outputs.cache-hit != 'true' && runner.os == 'macOS' + mkdir -p ~/.gmt + mv .gmt/* ~/.gmt + ls -lh ~/.gmt # Install the package that we want to test - name: Install the package From 19c38d0ddd3bd9b49808f925ca8011216095df85 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Wed, 22 Jul 2020 20:31:10 +1200 Subject: [PATCH 31/33] Move GMT Master Tests to separate yaml file and add a badge for it Also run the tests only on PRs that are ready for review or have a review requested (i.e. non-draft PRs). Based on https://github.community/t/dont-run-actions-on-draft-pull-requests/16817/6. --- .github/workflows/ci_tests.yaml | 65 ------------------------ .github/workflows/ci_tests_dev.yaml | 79 +++++++++++++++++++++++++++++ README.rst | 7 ++- 3 files changed, 84 insertions(+), 67 deletions(-) create mode 100644 .github/workflows/ci_tests_dev.yaml diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 252f96f7135..83e06e17595 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -122,68 +122,3 @@ jobs: file: ./coverage.xml # optional env_vars: OS,PYTHON fail_ci_if_error: true - - test_gmt_master: - name: GMT master - ${{ matrix.os }} - Python ${{ matrix.python-version }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - python-version: [3.7] - os: [ubuntu-20.04, macOS-10.15] - env: - # LD_LIBRARY_PATH: ${{ github.workspace }}/gmt/lib:$LD_LIBRARY_PATH - GMT_INSTALL_DIR: ${{ github.workspace }}/gmt-install-dir - GMT_LIBRARY_PATH: ${{ github.workspace }}/gmt-install-dir/lib - defaults: - run: - shell: bash -l {0} - - steps: - # Checkout current git repository - - name: Checkout - uses: actions/checkout@v2.3.1 - with: - # fecth all history so that versioneer works - fetch-depth: 0 - - # Setup Miniconda - - name: Setup Miniconda - uses: goanpeca/setup-miniconda@v1.6.0 - with: - python-version: ${{ matrix.python-version }} - channels: conda-forge - - # Install build dependencies from conda-forge - - name: Install build dependencies - run: conda install cmake libblas libcblas liblapack fftw gdal ghostscript libnetcdf hdf5 zlib curl pcre gshhg-gmt dcw-gmt ipython pytest pytest-cov pytest-mpl - - # Install GMT master branch - - name: Install GMT from master - run: curl https://raw.githubusercontent.com/GenericMappingTools/gmt/master/ci/build-gmt-master.sh | bash - - # Download cached remote files (artifacts) from Github - - name: Download remote data from Github - uses: dawidd6/action-download-artifact@v2.6.3 - with: - workflow: cache_data.yaml - name: gmt-cache - path: .gmt - - # Move downloaded files to ~/.gmt directory and list them - - name: Move and list downloaded remote files - shell: bash -l {0} - run: | - mkdir -p ~/.gmt - mv .gmt/* ~/.gmt - ls -lh ~/.gmt - - # Install the package that we want to test - - name: Install the package - run: | - python setup.py sdist --formats=zip - pip install dist/* - - # Run the tests - - name: Test with pytest - run: make test PYTEST_EXTRA="-r P" diff --git a/.github/workflows/ci_tests_dev.yaml b/.github/workflows/ci_tests_dev.yaml new file mode 100644 index 00000000000..7e516892638 --- /dev/null +++ b/.github/workflows/ci_tests_dev.yaml @@ -0,0 +1,79 @@ +# This workflow installs PyGMT dependencies, builds documentation and runs tests on GMT master +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: GMT Master Tests + +on: + # push: + # branches: [ master ] + pull_request: + types: [review_requested, ready_for_review] + # Schedule daily tests + schedule: + - cron: '0 0 * * *' + +jobs: + test_gmt_master: + name: ${{ matrix.os }} - Python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [3.8] + os: [ubuntu-20.04, macOS-10.15] + env: + # LD_LIBRARY_PATH: ${{ github.workspace }}/gmt/lib:$LD_LIBRARY_PATH + GMT_INSTALL_DIR: ${{ github.workspace }}/gmt-install-dir + GMT_LIBRARY_PATH: ${{ github.workspace }}/gmt-install-dir/lib + defaults: + run: + shell: bash -l {0} + + steps: + # Checkout current git repository + - name: Checkout + uses: actions/checkout@v2.3.1 + with: + # fecth all history so that versioneer works + fetch-depth: 0 + + # Setup Miniconda + - name: Setup Miniconda + uses: goanpeca/setup-miniconda@v1.6.0 + with: + python-version: ${{ matrix.python-version }} + channels: conda-forge + + # Install build dependencies from conda-forge + - name: Install build dependencies + run: conda install cmake libblas libcblas liblapack fftw gdal ghostscript libnetcdf hdf5 zlib curl pcre gshhg-gmt dcw-gmt ipython pytest pytest-cov pytest-mpl + + # Install GMT master branch + - name: Install GMT from master + run: curl https://raw.githubusercontent.com/GenericMappingTools/gmt/master/ci/build-gmt-master.sh | bash + + # Download cached remote files (artifacts) from Github + - name: Download remote data from Github + uses: dawidd6/action-download-artifact@v2.6.3 + with: + workflow: cache_data.yaml + name: gmt-cache + path: .gmt + + # Move downloaded files to ~/.gmt directory and list them + - name: Move and list downloaded remote files + shell: bash -l {0} + run: | + mkdir -p ~/.gmt + mv .gmt/* ~/.gmt + ls -lh ~/.gmt + + # Install the package that we want to test + - name: Install the package + run: | + python setup.py sdist --formats=zip + pip install dist/* + + # Run the tests + - name: Test with pytest + run: make test PYTEST_EXTRA="-r P" diff --git a/README.rst b/README.rst index 7d399e37489..8b56cfe997e 100644 --- a/README.rst +++ b/README.rst @@ -13,8 +13,11 @@ PyGMT :alt: Travis CI build status :target: https://travis-ci.org/GenericMappingTools/pygmt .. image:: https://github.com/GenericMappingTools/pygmt/workflows/Tests/badge.svg - :alt: GitHub Actions build status - :target: https://github.com/GenericMappingTools/pygmt/actions + :alt: GitHub Actions Tests status + :target: https://github.com/GenericMappingTools/pygmt/actions?query=workflow%3ATests +.. image:: https://github.com/GenericMappingTools/pygmt/workflows/GMT%20Master%20Tests/badge.svg + :alt: GitHub Actions GMT Master Tests status + :target: https://github.com/GenericMappingTools/pygmt/actions?query=workflow%3A"GMT+Master+Tests" .. image:: https://img.shields.io/codecov/c/github/GenericMappingTools/pygmt/master.svg?style=flat-square :alt: Test coverage status :target: https://codecov.io/gh/GenericMappingTools/pygmt From b91136697dfd0abb836f0593c332e16a8b3911f3 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Wed, 22 Jul 2020 17:30:12 -0400 Subject: [PATCH 32/33] No need to install gshhg-gmt and dcw-gmt --- .github/workflows/ci_tests_dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests_dev.yaml b/.github/workflows/ci_tests_dev.yaml index 7e516892638..16ef1d6eb59 100644 --- a/.github/workflows/ci_tests_dev.yaml +++ b/.github/workflows/ci_tests_dev.yaml @@ -46,7 +46,7 @@ jobs: # Install build dependencies from conda-forge - name: Install build dependencies - run: conda install cmake libblas libcblas liblapack fftw gdal ghostscript libnetcdf hdf5 zlib curl pcre gshhg-gmt dcw-gmt ipython pytest pytest-cov pytest-mpl + run: conda install cmake libblas libcblas liblapack fftw gdal ghostscript libnetcdf hdf5 zlib curl pcre ipython pytest pytest-cov pytest-mpl # Install GMT master branch - name: Install GMT from master From c174a1e824f40e85d9385fe31f5ebdffddc38d42 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Thu, 23 Jul 2020 10:03:22 +1200 Subject: [PATCH 33/33] Mention GMT Master Tests in MAINTENANCE.md, and reorganize some sections --- MAINTENANCE.md | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/MAINTENANCE.md b/MAINTENANCE.md index 37b90a511ce..9b560e68013 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -42,16 +42,33 @@ The main advantages of this are: We use Github Actions and TravisCI continuous integration (CI) services to build and test the project on Linux, macOS and Windows. -The configuration file for these services are in -`.github/workflows/ci_tests.yaml` and `.travis.yml`. They rely on the `requirements.txt` file to install required dependencies using conda and the `Makefile` to run the tests and checks. -There is also a `.github/workflows/cache_data.yaml` file which caches some GMT -remote data files needed for the Github Actions CI on a regular basis. + +### Github Actions + +There are 3 configuration files located in `.github/workflows`: + +1. `ci_tests.yaml` (Style Checks, Tests on Linux/macOS/Windows) + +This is ran on every commit on the *master* and Pull Request branches. +It is also scheduled to run daily on the *master* branch. + +2. `ci_tests_dev.yaml` (GMT Master Tests on Linux/macOS). + +This is only triggered when a review is requested or re-requested on a PR. +It is also scheduled to run daily on the *master* branch. + +3. `cache_data.yaml` (Caches GMT remote data files needed for Github Actions CI) + +This is scheduled to run every Sunday at 12 noon. If new remote files are needed urgently, maintainers can manually uncomment the 'pull_request:' line in that `cache_data.yaml` file to refresh the cache. -Travis also handles all of our deployments automatically: +### Travis CI + +The configuration file is at `.travis.yml`. +Travis runs tests (Linux only) and handles all of our deployments automatically: * Updating the development documentation by pushing the built HTML pages from the *master* branch onto the `dev` folder of the *gh-pages* branch.