From 301d9d4be95c300ae3cd3a6fcc6c81f422cfee5a Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Wed, 10 Jul 2024 11:19:36 -0400 Subject: [PATCH 01/14] ci: added action for testing tools ci: testing tools from master branch ci: handling segfaults from imagej stuff fix: trap invocation fix: fewer cases in trap invocation fix: trap now exits 0 after catching seg-fault ci: using conda for mm-tools ci: movinf poetry config to its own step fix: no more $ ci: ignoring mm-tools jobs until after cwl_utilities is available ci: checking tools after pinning numpy under 2.0 chore: triggering run ci: triggering run 2 fix: using workflow from new branch docs: added comment explaining the trap command ci: triggering run 3 ci: using new github action plugin for conda using tools from main repo again --- .github/workflows/test_tools.yml | 267 +++++++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 .github/workflows/test_tools.yml diff --git a/.github/workflows/test_tools.yml b/.github/workflows/test_tools.yml new file mode 100644 index 0000000..99197e7 --- /dev/null +++ b/.github/workflows/test_tools.yml @@ -0,0 +1,267 @@ +name: Test Image/Tabular/MM Tools + +on: + workflow_dispatch: + pull_request: + push: + branches-ignore: + - master + +jobs: + build_wheels: + name: Build Wheels + runs-on: ubuntu-latest + steps: + - name: Checkout 🛎️ + uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: '3.9' + - name: Install build tools + run: | + python -m pip install twine build + - name: Build wheels + run: | + python -m build + - name: Upload Artifacts + uses: actions/upload-artifact@v4 + with: + name: wheels + path: dist/*.whl + retention-days: 1 + + find_image_tools: + needs: build_wheels + name: Find Image Tools + uses: PolusAI/image-tools/.github/workflows/find-all-tools.yml@master + + image-tests: + name: Images | ${{ matrix.tool_name }} + runs-on: ubuntu-latest + needs: find_image_tools + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.find_image_tools.outputs.matrix) }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + repository: 'PolusAI/image-tools' + ref: "master" + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: '3.9' + - name: Conda | Install + uses: conda-incubator/setup-miniconda@v3 + - name: Poetry | Install + uses: abatilo/actions-poetry@v3 + - name: Poetry | Config + run: | + poetry config virtualenvs.create false + - name: Tests | Pre-Update + run: | + # We see seg-faults during the cleanup phase of running test on the imagej tools. + set -e + trap 'case $? in + 139) echo "::warning::Seg-fault detected in ${{ matrix.tool_name }}. Please check the logs for more information." && exit 0;; + esac' ERR + + cd ${{ matrix.tool_dir }} + + if [ -f "environment.yml" ]; then + conda init bash + source ~/.bashrc + conda env create -f environment.yml + conda activate project_env + pip install -e ".[all]" + conda install pytest + python -m pytest -v + else + poetry install + poetry run pytest -v + fi + - name: Download Wheels + uses: actions/download-artifact@v4 + with: + name: wheels + path: dist + - name: Tests | Post-Update + run: | + set -e + trap 'case $? in + 139) echo "::warning::Seg-fault detected in ${{ matrix.tool_name }}. Please check the logs for more information." && exit 0;; + esac' ERR + + cd ${{ matrix.tool_dir }} + + if [ -f "environment.yml" ]; then + conda init bash + source ~/.bashrc + conda activate project_env + pip install -e ".[all]" + pip install --find-links=${root_dir}/dist bfio + conda install pytest + python -m pytest -v + else + poetry install + pip install --find-links=${root_dir}/dist bfio + poetry run pytest -v + fi + + find_tabular_tools: + needs: build_wheels + name: Find Tabular Tools + uses: PolusAI/tabular-tools/.github/workflows/find-all-tools.yml@main + + tabular-tests: + name: Tabular | ${{ matrix.tool_name }} + runs-on: ubuntu-latest + needs: find_tabular_tools + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.find_tabular_tools.outputs.matrix) }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + repository: 'PolusAI/tabular-tools' + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: '3.9' + - name: Conda | Install + uses: conda-incubator/setup-miniconda@v3 + - name: Poetry | Install + uses: abatilo/actions-poetry@v3 + - name: Poetry | Config + run: | + poetry config virtualenvs.create false + - name: Tests | Pre-Update + run: | + set -e + trap 'case $? in + 139) echo "::warning::Seg-fault detected in ${{ matrix.tool_name }}. Please check the logs for more information." && exit 0;; + esac' ERR + + cd ${{ matrix.tool_dir }} + + if [ -f "environment.yml" ]; then + conda init bash + source ~/.bashrc + conda env create -f environment.yml + conda activate project_env + pip install -e ".[all]" + conda install pytest + python -m pytest -v + else + poetry install + poetry run pytest -v + fi + - name: Download Wheels + uses: actions/download-artifact@v4 + with: + name: wheels + path: dist + - name: Tests | Post-Update + run: | + set -e + trap 'case $? in + 139) echo "::warning::Seg-fault detected in ${{ matrix.tool_name }}. Please check the logs for more information." && exit 0;; + esac' ERR + + cd ${{ matrix.tool_dir }} + + if [ -f "environment.yml" ]; then + conda init bash + source ~/.bashrc + conda activate project_env + pip install -e ".[all]" + pip install --find-links=${root_dir}/dist bfio + conda install pytest + python -m pytest -v + else + poetry install + pip install --find-links=${root_dir}/dist bfio + poetry run pytest -v + fi + + # TODO: Uncomment tests for mm-tools after the pip package for `cwl_utilities` + # is available on PyPI and the PRs on mm-tools are merged. + # find_mm_tools: + # needs: build_wheels + # name: Find MM Tools + # uses: PolusAI/mm-tools/.github/workflows/find-all-tools.yml@main + + # mm-tests: + # name: MM | ${{ matrix.tool_name }} + # runs-on: ubuntu-latest + # needs: find_mm_tools + # strategy: + # fail-fast: false + # matrix: ${{ fromJson(needs.find_mm_tools.outputs.matrix) }} + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # repository: 'PolusAI/mm-tools' + # - name: Set up Python 3.9 + # uses: actions/setup-python@v5 + # with: + # python-version: '3.9' + # - name: Conda | Install + # uses: conda-incubator/setup-miniconda@v3 + # - name: Poetry | Install + # uses: abatilo/actions-poetry@v3 + # - name: Poetry | Config + # run: | + # poetry config virtualenvs.create false + # - name: Tests | Pre-Update + # run: | + # set -e + # trap 'case $? in + # 139) echo "::warning::Seg-fault detected in ${{ matrix.tool_name }}. Please check the logs for more information." && exit 0;; + # esac' ERR + + # cd ${{ matrix.tool_dir }} + + # if [ -f "environment.yml" ]; then + # conda init bash + # source ~/.bashrc + # conda env create -f environment.yml + # conda activate project_env + # pip install -e ".[all]" + # conda install pytest + # python -m pytest -v + # else + # poetry install + # poetry run pytest -v + # fi + # - name: Download Wheels + # uses: actions/download-artifact@v4 + # with: + # name: wheels + # path: dist + # - name: Tests | Post-Update + # run: | + # set -e + # trap 'case $? in + # 139) echo "::warning::Seg-fault detected in ${{ matrix.tool_name }}. Please check the logs for more information." && exit 0;; + # esac' ERR + + # cd ${{ matrix.tool_dir }} + + # if [ -f "environment.yml" ]; then + # conda init bash + # source ~/.bashrc + # conda activate project_env + # pip install -e ".[all]" + # pip install --find-links=${root_dir}/dist bfio + # conda install pytest + # python -m pytest -v + # else + # poetry install + # pip install --find-links=${root_dir}/dist bfio + # poetry run pytest -v + # fi From c0825885d80ef8ce096d59ad4a1a3e97f6a2dcd1 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Mon, 15 Jul 2024 10:35:02 -0400 Subject: [PATCH 02/14] fix: forcing reinstall of bfio from wheel --- .github/workflows/test_tools.yml | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test_tools.yml b/.github/workflows/test_tools.yml index 99197e7..2ac8d75 100644 --- a/.github/workflows/test_tools.yml +++ b/.github/workflows/test_tools.yml @@ -101,14 +101,12 @@ jobs: source ~/.bashrc conda activate project_env pip install -e ".[all]" - pip install --find-links=${root_dir}/dist bfio conda install pytest - python -m pytest -v else poetry install - pip install --find-links=${root_dir}/dist bfio - poetry run pytest -v - fi + fi + pip install --upgrade --no-deps --force-reinstall --find-links=${root_dir}/dist bfio + poetry run pytest -v find_tabular_tools: needs: build_wheels @@ -178,14 +176,12 @@ jobs: source ~/.bashrc conda activate project_env pip install -e ".[all]" - pip install --find-links=${root_dir}/dist bfio conda install pytest - python -m pytest -v else poetry install - pip install --find-links=${root_dir}/dist bfio - poetry run pytest -v - fi + fi + pip install --upgrade --no-deps --force-reinstall --find-links=${root_dir}/dist bfio + poetry run pytest -v # TODO: Uncomment tests for mm-tools after the pip package for `cwl_utilities` # is available on PyPI and the PRs on mm-tools are merged. @@ -257,11 +253,9 @@ jobs: # source ~/.bashrc # conda activate project_env # pip install -e ".[all]" - # pip install --find-links=${root_dir}/dist bfio # conda install pytest - # python -m pytest -v # else # poetry install - # pip install --find-links=${root_dir}/dist bfio - # poetry run pytest -v - # fi + # fi + # pip install --upgrade --no-deps --force-reinstall --find-links=${root_dir}/dist bfio + # poetry run pytest -v From d1d531c4a53af840d47d5d56d862d144bb48cdb0 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Mon, 15 Jul 2024 10:40:07 -0400 Subject: [PATCH 03/14] fix: removed no-deps flag from bfio reinstall --- .github/workflows/test_tools.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_tools.yml b/.github/workflows/test_tools.yml index 2ac8d75..18eb3a2 100644 --- a/.github/workflows/test_tools.yml +++ b/.github/workflows/test_tools.yml @@ -105,7 +105,7 @@ jobs: else poetry install fi - pip install --upgrade --no-deps --force-reinstall --find-links=${root_dir}/dist bfio + pip install --upgrade --force-reinstall --find-links=${root_dir}/dist bfio poetry run pytest -v find_tabular_tools: @@ -180,7 +180,7 @@ jobs: else poetry install fi - pip install --upgrade --no-deps --force-reinstall --find-links=${root_dir}/dist bfio + pip install --upgrade --force-reinstall --find-links=${root_dir}/dist bfio poetry run pytest -v # TODO: Uncomment tests for mm-tools after the pip package for `cwl_utilities` @@ -257,5 +257,5 @@ jobs: # else # poetry install # fi - # pip install --upgrade --no-deps --force-reinstall --find-links=${root_dir}/dist bfio + # pip install --upgrade --force-reinstall --find-links=${root_dir}/dist bfio # poetry run pytest -v From 650c74974accbd81cbc43bf6b5b9392278d4f3b9 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Tue, 16 Jul 2024 10:02:13 -0400 Subject: [PATCH 04/14] wip: triggering actions From 79d20f39574c59cc30e9c9b52e2485582b8ae063 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Mon, 22 Jul 2024 10:10:48 -0400 Subject: [PATCH 05/14] ci: triggering actions From a61e20c746a858d3f5431a663bac29fd6d7b79f0 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Mon, 22 Jul 2024 10:17:10 -0400 Subject: [PATCH 06/14] ci: using correct root dir for forcing bfio re-installation --- .github/workflows/test_tools.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_tools.yml b/.github/workflows/test_tools.yml index 18eb3a2..2855d1f 100644 --- a/.github/workflows/test_tools.yml +++ b/.github/workflows/test_tools.yml @@ -94,6 +94,7 @@ jobs: 139) echo "::warning::Seg-fault detected in ${{ matrix.tool_name }}. Please check the logs for more information." && exit 0;; esac' ERR + root_dir=$(pwd) cd ${{ matrix.tool_dir }} if [ -f "environment.yml" ]; then @@ -169,6 +170,7 @@ jobs: 139) echo "::warning::Seg-fault detected in ${{ matrix.tool_name }}. Please check the logs for more information." && exit 0;; esac' ERR + root_dir=$(pwd) cd ${{ matrix.tool_dir }} if [ -f "environment.yml" ]; then @@ -246,6 +248,7 @@ jobs: # 139) echo "::warning::Seg-fault detected in ${{ matrix.tool_name }}. Please check the logs for more information." && exit 0;; # esac' ERR + # root_dir=$(pwd) # cd ${{ matrix.tool_dir }} # if [ -f "environment.yml" ]; then From 4e9d82842abdc6f3ee7e8dc5bda80a1f92a46d57 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Mon, 22 Jul 2024 10:27:50 -0400 Subject: [PATCH 07/14] ci: trying different install command for bfio wheel --- .github/workflows/test_tools.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_tools.yml b/.github/workflows/test_tools.yml index 2855d1f..93b80af 100644 --- a/.github/workflows/test_tools.yml +++ b/.github/workflows/test_tools.yml @@ -106,7 +106,7 @@ jobs: else poetry install fi - pip install --upgrade --force-reinstall --find-links=${root_dir}/dist bfio + pip install --force-reinstall --find-links=${root_dir}/dist bfio poetry run pytest -v find_tabular_tools: @@ -182,7 +182,7 @@ jobs: else poetry install fi - pip install --upgrade --force-reinstall --find-links=${root_dir}/dist bfio + pip install --force-reinstall --find-links=${root_dir}/dist bfio poetry run pytest -v # TODO: Uncomment tests for mm-tools after the pip package for `cwl_utilities` From 60f80096a5b0ed1a383410ea35586b6e556a10d6 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Thu, 25 Jul 2024 11:23:37 -0400 Subject: [PATCH 08/14] ci: triggering actions From 89959c817dd1f3804c267cc653e66a8316fe8815 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Thu, 25 Jul 2024 11:46:56 -0400 Subject: [PATCH 09/14] ci: relying on poetry to resolve deps between numpy<2.0 and bfio wheel --- .github/workflows/test_tools.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_tools.yml b/.github/workflows/test_tools.yml index 93b80af..d13037e 100644 --- a/.github/workflows/test_tools.yml +++ b/.github/workflows/test_tools.yml @@ -95,6 +95,8 @@ jobs: esac' ERR root_dir=$(pwd) + bfio_wheel=$(ls ${root_dir}/dist/bfio*.whl) + echo "Installing bfio from ${bfio_wheel}" cd ${{ matrix.tool_dir }} if [ -f "environment.yml" ]; then @@ -106,7 +108,9 @@ jobs: else poetry install fi - pip install --force-reinstall --find-links=${root_dir}/dist bfio + poetry remove bfio + poetry add "numpy<2.0" + poetry add "${bfio_wheel}[all]" poetry run pytest -v find_tabular_tools: @@ -182,7 +186,9 @@ jobs: else poetry install fi - pip install --force-reinstall --find-links=${root_dir}/dist bfio + poetry remove bfio + poetry add "numpy<2.0" + poetry add "${bfio_wheel}[all]" poetry run pytest -v # TODO: Uncomment tests for mm-tools after the pip package for `cwl_utilities` From 53fa4a91a16cfb13e2af85a8daa74ee068525bba Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Thu, 25 Jul 2024 13:34:00 -0400 Subject: [PATCH 10/14] ci: no longer trying to remove bfio --- .github/workflows/test_tools.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test_tools.yml b/.github/workflows/test_tools.yml index d13037e..0714a17 100644 --- a/.github/workflows/test_tools.yml +++ b/.github/workflows/test_tools.yml @@ -108,7 +108,6 @@ jobs: else poetry install fi - poetry remove bfio poetry add "numpy<2.0" poetry add "${bfio_wheel}[all]" poetry run pytest -v @@ -186,7 +185,6 @@ jobs: else poetry install fi - poetry remove bfio poetry add "numpy<2.0" poetry add "${bfio_wheel}[all]" poetry run pytest -v From fefffa95870430e6d4d1fd5c4c30a93dd2a09bff Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Thu, 25 Jul 2024 13:47:15 -0400 Subject: [PATCH 11/14] ci: fixing post-update step for tabular-tools --- .github/workflows/test_tools.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_tools.yml b/.github/workflows/test_tools.yml index 0714a17..4544b8d 100644 --- a/.github/workflows/test_tools.yml +++ b/.github/workflows/test_tools.yml @@ -174,6 +174,8 @@ jobs: esac' ERR root_dir=$(pwd) + bfio_wheel=$(ls ${root_dir}/dist/bfio*.whl) + echo "Installing bfio from ${bfio_wheel}" cd ${{ matrix.tool_dir }} if [ -f "environment.yml" ]; then From 697c4989d06da316d2547d1bd330c3d45d19e631 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Thu, 25 Jul 2024 13:48:33 -0400 Subject: [PATCH 12/14] wip: changing repo for tests --- .github/workflows/test_tools.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_tools.yml b/.github/workflows/test_tools.yml index 4544b8d..ec21ef7 100644 --- a/.github/workflows/test_tools.yml +++ b/.github/workflows/test_tools.yml @@ -34,7 +34,7 @@ jobs: find_image_tools: needs: build_wheels name: Find Image Tools - uses: PolusAI/image-tools/.github/workflows/find-all-tools.yml@master + uses: nishaq503/image-tools/.github/workflows/find-all-tools.yml@fix/typo image-tests: name: Images | ${{ matrix.tool_name }} @@ -47,8 +47,8 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - repository: 'PolusAI/image-tools' - ref: "master" + repository: 'nishaq503/image-tools' + ref: "fix/typo" - name: Set up Python 3.9 uses: actions/setup-python@v5 with: @@ -115,7 +115,7 @@ jobs: find_tabular_tools: needs: build_wheels name: Find Tabular Tools - uses: PolusAI/tabular-tools/.github/workflows/find-all-tools.yml@main + uses: nishaq503/tabular-tools/.github/workflows/find-all-tools.yml@fix/typo tabular-tests: name: Tabular | ${{ matrix.tool_name }} @@ -128,7 +128,8 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - repository: 'PolusAI/tabular-tools' + repository: 'nishaq503/tabular-tools' + ref: "fix/typo" - name: Set up Python 3.9 uses: actions/setup-python@v5 with: From 67d54d051c4aca27d7fdb55ef4c989059c86b4be Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Thu, 25 Jul 2024 14:01:16 -0400 Subject: [PATCH 13/14] ci: triggering actions From e1c6c22386a2bdcda92e5d3fdc18d0a7a5280842 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Thu, 25 Jul 2024 14:14:31 -0400 Subject: [PATCH 14/14] ci: resetting to main repo --- .github/workflows/test_tools.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_tools.yml b/.github/workflows/test_tools.yml index ec21ef7..5b5b334 100644 --- a/.github/workflows/test_tools.yml +++ b/.github/workflows/test_tools.yml @@ -34,7 +34,7 @@ jobs: find_image_tools: needs: build_wheels name: Find Image Tools - uses: nishaq503/image-tools/.github/workflows/find-all-tools.yml@fix/typo + uses: PolusAI/image-tools/.github/workflows/find-all-tools.yml@master image-tests: name: Images | ${{ matrix.tool_name }} @@ -47,8 +47,8 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - repository: 'nishaq503/image-tools' - ref: "fix/typo" + repository: 'PolusAI/image-tools' + ref: "master" - name: Set up Python 3.9 uses: actions/setup-python@v5 with: @@ -115,7 +115,7 @@ jobs: find_tabular_tools: needs: build_wheels name: Find Tabular Tools - uses: nishaq503/tabular-tools/.github/workflows/find-all-tools.yml@fix/typo + uses: PolusAI/tabular-tools/.github/workflows/find-all-tools.yml@main tabular-tests: name: Tabular | ${{ matrix.tool_name }} @@ -128,8 +128,8 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - repository: 'nishaq503/tabular-tools' - ref: "fix/typo" + repository: 'PolusAI/tabular-tools' + ref: "main" - name: Set up Python 3.9 uses: actions/setup-python@v5 with: