Parallelize pytest (#1)

mashehu · Nov 30, 2023 · 23295ac · 23295ac
1 parent 20dbeec
commit 23295ac
Show file tree

Hide file tree

Showing 5 changed files with 71 additions and 29 deletions.
diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml
@@ -17,7 +17,7 @@ env:
 
 jobs:
   MakeTestWorkflow:
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     env:
       NXF_ANSI_LOG: false
     strategy:

diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml
@@ -14,7 +14,7 @@ concurrency:
 
 jobs:
   EditorConfig:
-    runs-on: ["self-hosted"]
+    runs-on: ["ubuntu-latest"]
     steps:
       - uses: actions/checkout@v4
 
@@ -30,7 +30,7 @@ jobs:
         run: editorconfig-checker -exclude README.md $(git ls-files | grep -v 'test\|.py\|md\|json\|yml\|yaml\|html\|css\|Makefile')
 
   Prettier:
-    runs-on: ["self-hosted"]
+    runs-on: ["ubuntu-latest"]
     steps:
       - uses: actions/checkout@v4
 
@@ -45,7 +45,7 @@ jobs:
         run: prettier --check ${GITHUB_WORKSPACE}
 
   PythonBlack:
-    runs-on: ["self-hosted"]
+    runs-on: ["ubuntu-latest"]
     steps:
       - uses: actions/checkout@v4
 
@@ -75,7 +75,7 @@ jobs:
           allow-repeats: false
 
   isort:
-    runs-on: ["self-hosted"]
+    runs-on: ["ubuntu-latest"]
     steps:
       - name: Check out source-code repository
         uses: actions/checkout@v4
@@ -91,7 +91,7 @@ jobs:
           requirementsFiles: "requirements.txt requirements-dev.txt"
 
   static-type-check:
-    runs-on: ["self-hosted"]
+    runs-on: ["ubuntu-latest"]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4

diff --git a/.github/workflows/push_dockerhub_dev.yml b/.github/workflows/push_dockerhub_dev.yml
@@ -13,7 +13,7 @@ concurrency:
 jobs:
   push_dockerhub:
     name: Push new Docker image to Docker Hub (dev)
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     # Only run for the nf-core repo, for releases and merged PRs
     if: ${{ github.repository == 'nf-core/tools' }}
     env:

diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -14,8 +14,9 @@ on:
       - "CHANGELOG.md"
   release:
     types: [published]
+  workflow_dispatch:
 
-# Cancel if a newer run is started
+# Cancel if a newer run with the same workflow name is queued
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -39,32 +40,42 @@ jobs:
         id: conditions
         run: echo "run-tests=${{ github.ref == 'refs/heads/master' || (matrix.runner == 'ubuntu-20.04' && matrix.python-version == '3.8') }}" >> $GITHUB_ENV
 
-      - name: Check out source-code repository
-        uses: actions/checkout@v4
-        if: ${{ env.run-tests == 'true' }}
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-        if: ${{ env.run-tests == 'true' }}
     outputs:
       python-version: ${{ matrix.python-version }}
       runner: ${{ matrix.runner }}
       run-tests: ${{ env.run-tests }}
 
+  # create a test matrix based on all python files in /tests
+  list_tests:
+    name: Get test file matrix
+    needs: setup
+    if: ${{ needs.setup.outputs.run-tests == 'true' }}
+    runs-on: ${{ needs.setup.outputs.runner }}
+    outputs:
+      tests: ${{ env.tests }}
+    steps:
+      - uses: actions/checkout@v4
+        name: Check out source-code repository
+
+      - name: List tests
+        id: list_tests
+        run:
+          | # get all python files in /tests except __init__.py and also all subdirectories and remove trailing commas
+          echo "tests=$(find tests/test_* | sed 's/tests\///g' | jq -R -s -c '{test: (split("\n")[:-1])}')" >> $GITHUB_ENV
+
   test:
     name: Test with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }}
-    needs: setup
+    needs: [setup, list_tests]
     if: ${{ needs.setup.outputs.run-tests == 'true' }}
     runs-on: ${{ needs.setup.outputs.runner }}
+    strategy:
+      matrix: ${{ fromJson(needs.list_tests.outputs.tests) }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         name: Check out source-code repository
 
       - name: Set up Python ${{ needs.setup.outputs.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ needs.setup.outputs.python-version }}
           cache: "pip"
@@ -87,8 +98,6 @@ jobs:
 
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v1
-        with:
-          version: "latest-everything"
 
       - name: Cache nf-test installation
         id: cache-software
@@ -106,10 +115,43 @@ jobs:
           sudo mv nf-test /usr/local/bin/
 
       - name: Test with pytest
-        run: python3 -m pytest tests/ --color=yes --cov-report=xml --cov-config=.github/.coveragerc --cov=nf_core
+        run: |
+          python3 -m pytest tests/${{matrix.test}} --color=yes --cov-report=xml --cov-config=.github/.coveragerc --cov=nf_core || exit_code=$?
+          # don't fail if no tests were collected, e.g. for test_licence.py
+          if [ "${exit_code}" -eq 5 ]; then
+            echo "No tests were collected"
+            exit 0
+          elif [ "${exit_code}" -ne 0 ]; then
+            echo "Tests failed with exit code ${exit_code}"
+            exit 1
+          fi
+
+      - name: Upload coverage
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage${{ matrix.test }}
+          path: .coverage
 
-      - uses: codecov/codecov-action@v1
-        name: Upload code coverage report
+  coverage:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
         with:
-          if: success()
-          token: ${{ secrets.CODECOV_TOKEN }}
+          python-version: 3.12
+          cache: "pip"
+      - name: Install deps
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      - name: Download all artifacts
+        # Downloads coverage1, coverage2, etc.
+        uses: actions/download-artifact@v3
+      - name: Run coverage
+        run: |
+          coverage combine coverage*/.coverage*
+          coverage report
+          coverage xml
+      - uses: codecov/codecov-action@v3
diff --git a/.github/workflows/tools-api-docs-dev.yml b/.github/workflows/tools-api-docs-dev.yml
@@ -20,7 +20,7 @@ concurrency:
 jobs:
   api-docs:
     name: Build & push Sphinx API docs
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
 
     steps:
       - name: Check out source-code repository