Add conditional coverage flags for accurate platform coverage

.github/workflows/ci.yml

@@ -29,25 +29,25 @@ jobs:
     uses: beeware/.github/.github/workflows/towncrier-run.yml@main
 
   package:
-    name: Python Package
+    name: Python package
     uses: beeware/.github/.github/workflows/python-package-create.yml@main
 
   unit-tests:
     name: Unit tests
-    needs: [pre-commit, towncrier, package]
+    needs: [ pre-commit, towncrier, package ]
     runs-on: ${{ matrix.platform }}-latest
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        platform: [ "macos", "ubuntu", "windows" ]
+        platform: [ "macOS", "Ubuntu", "Windows" ]
         python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12-dev" ]
         include:
         - experimental: false
         # Allow dev Python to fail without failing entire job
         - python-version: "3.12-dev"
           experimental: true
         # Run tests against the latest Windows Store Python
-        - platform: "windows"
+        - platform: "Windows"
           python-version: "winstore"
           experimental: false
     steps:
@@ -57,52 +57,130 @@ jobs:
         fetch-depth: 0
 
     - name: Set up Python
-      if: matrix.python-version != 'winstore'
+      if: startswith(matrix.python-version, '3')
       uses: actions/[email protected]
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Install Windows Store Python
       if: matrix.python-version == 'winstore'
       uses: beeware/.github/.github/actions/install-win-store-python@main
-      with:
-        python-version: "3.11"
 
-    - name: Get packages
+    - name: Get Packages
       uses: actions/[email protected]
       with:
         name: ${{ needs.package.outputs.artifact-name }}
         path: dist
 
-    - name: Install dev dependencies
+    - name: Install dev Dependencies
       run: |
-        # pip 23.1 has an issue with --user installs.
-        # See https://github.com/pypa/pip/issues/11982 for details
-        python -m pip install --upgrade "pip!=23.1"
+        python -m pip install --upgrade pip
         python -m pip install --upgrade setuptools
         # We don't actually want to install briefcase; we just
         # want the dev extras so we have a known version of tox.
         python -m pip install $(ls dist/briefcase-*.whl)[dev]
 
     - name: Test
+      id: test
+      env:
+        COVERAGE_FILE: ".coverage.${{ matrix.platform }}.${{ matrix.python-version }}"
       run: tox -e py --installpkg dist/briefcase-*.whl
 
-    - name: Store coverage data
+    - name: Store Coverage Data
+      if: always() && (steps.test.outcome == 'failure' || steps.test.outcome == 'success')
       uses: actions/[email protected]
       with:
         name: coverage-data
         path: ".coverage.*"
         if-no-files-found: ignore
 
-    - name: Report platform coverage
-      run: tox -e coverage
+  coverage-platform:
+    name: Platform coverage - ${{ matrix.platform }}
+    runs-on: ${{ matrix.platform }}-latest
+    needs: unit-tests
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ "macOS", "Ubuntu", "Windows" ]
+    steps:
+    - name: Checkout
+      uses: actions/[email protected]
+      with:
+        fetch-depth: 0
+
+    - name: Setup Python
+      uses: actions/[email protected]
+      with:
+        python-version: |
+          3.12-dev
+          3.11
+          3.10
+          3.9
+          3.8
+
+    - name: Install dev Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade setuptools
+        # We don't actually want to install briefcase; we just
+        # want the dev extras so we have a known version of tox.
+        python -m pip install -e .[dev]
+
+    - name: Retrieve Coverage Data
+      uses: actions/[email protected]
+      with:
+        name: coverage-data
 
-  coverage:
-    name: Combine & check coverage.
+    - name: ${{ matrix.platform }} Coverage Report
+      env:
+        COVERAGE_FILE: ".coverage.${{ matrix.platform }}"
+      run: tox -qe coverage-platform-html-keep
+
+    - name: Python 3.12 on ${{ matrix.platform }} Coverage Report
+      if: success() || failure()
+      continue-on-error: true
+      env:
+        COVERAGE_FILE: ".coverage.${{ matrix.platform }}.3.12-dev"
+      run: tox -qe coverage312-html-keep
+
+    - name: Python 3.11 on ${{ matrix.platform }} Coverage Report
+      if: success() || failure()
+      env:
+        COVERAGE_FILE: ".coverage.${{ matrix.platform }}.3.11"
+      run: tox -qe coverage311-html-keep
+
+    - name: Python 3.10 on ${{ matrix.platform }} Coverage Report
+      if: success() || failure()
+      env:
+        COVERAGE_FILE: ".coverage.${{ matrix.platform }}.3.10"
+      run: tox -qe coverage310-html-keep
+
+    - name: Python 3.9 on ${{ matrix.platform }} Coverage Report
+      if: success() || failure()
+      env:
+        COVERAGE_FILE: ".coverage.${{ matrix.platform }}.3.9"
+      run: tox -qe coverage39-html-keep
+
+    - name: Python 3.8 on ${{ matrix.platform }} Coverage Report
+      if: success() || failure()
+      env:
+        COVERAGE_FILE: ".coverage.${{ matrix.platform }}.3.8"
+      run: tox -qe coverage38-html-keep
+
+    - name: Upload HTML Coverage Report
+      if: failure()
+      uses: actions/[email protected]
+      with:
+        name: html-platform-coverage-report-${{ matrix.platform }}
+        path: htmlcov
+
+  coverage-project:
+    name: Project coverage
     runs-on: ubuntu-latest
     needs: unit-tests
     steps:
-    - uses: actions/[email protected]
+    - name: Checkout
+      uses: actions/[email protected]
       with:
         fetch-depth: 0
 
@@ -113,33 +191,32 @@ jobs:
         # https://github.com/nedbat/coveragepy/issues/1572#issuecomment-1522546425
         python-version: "3.8"
 
-    - name: Install dev dependencies
+    - name: Install dev Dependencies
       run: |
-        # pip 23.1 has an issue with --user installs.
-        # See https://github.com/pypa/pip/issues/11982 for details
-        python -m pip install --upgrade "pip!=23.1"
+        python -m pip install --upgrade pip
         python -m pip install --upgrade setuptools
         # We don't actually want to install briefcase; we just
         # want the dev extras so we have a known version of tox.
         python -m pip install -e .[dev]
 
-    - name: Retrieve coverage data
+    - name: Retrieve Coverage Data
       uses: actions/[email protected]
       with:
         name: coverage-data
 
-    - name: Generate coverage report
-      run: tox -e coverage-html-fail
+    - name: Project Coverage Report
+      id: coverage
+      run: tox -qe coverage-project-html
 
-    - name: Upload HTML report if check failed.
-      if: ${{ failure() }}
+    - name: Upload HTML Coverage Report
+      if: always() && steps.coverage.outcome == 'failure'
       uses: actions/[email protected]
       with:
-        name: html-coverage-report
+        name: html-project-coverage-report
         path: htmlcov
 
   verify-apps:
-    name: Build App
+    name: Build app
     needs: unit-tests
     uses: beeware/.github/.github/workflows/app-build-verify.yml@main
     with:

changes/1260.misc.rst

@@ -0,0 +1 @@
+Coverage reporting for a specific versions of Python or a specific platform is now supported.

pyproject.toml

@@ -3,6 +3,7 @@ requires = ["setuptools>=60", "setuptools_scm[toml]>=7.0"]
 build-backend = "setuptools.build_meta"
 
 [tool.coverage.run]
+plugins = ["coverage_conditional_plugin"]
 parallel = true
 branch = true
 relative_files = true
@@ -26,6 +27,37 @@ exclude_lines = [
     "if TYPE_CHECKING:",
 ]
 
+[tool.coverage.coverage_conditional_plugin.rules]
+# Packages/Modules
+no-cover-if-missing-setuptools_scm = "not is_installed('setuptools_scm')"
+# Linux
+no-cover-if-is-linux = "sys_platform == 'linux' and os_environ.get('COVERAGE_EXCLUDE_PLATFORM') != 'disable'"
+no-cover-if-not-linux = "sys_platform != 'linux' and os_environ.get('COVERAGE_EXCLUDE_PLATFORM') != 'disable'"
+# macOS
+no-cover-if-is-macos = "sys_platform == 'darwin' and os_environ.get('COVERAGE_EXCLUDE_PLATFORM') != 'disable'"
+no-cover-if-not-macos = "sys_platform != 'darwin' and os_environ.get('COVERAGE_EXCLUDE_PLATFORM') != 'disable'"
+# Windows
+no-cover-if-is-windows = "sys_platform == 'win32' and os_environ.get('COVERAGE_EXCLUDE_PLATFORM') != 'disable'"
+no-cover-if-not-windows = "sys_platform != 'win32' and os_environ.get('COVERAGE_EXCLUDE_PLATFORM') != 'disable'"
+# Python 3.12
+no-cover-if-is-py312 = "python_version == '3.12' and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
+no-cover-if-lt-py312 = "sys_version_info < (3, 12) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
+no-cover-if-gte-py312 = "sys_version_info > (3, 12) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
+# Python 3.11
+no-cover-if-is-py311 = "python_version == '3.11' and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
+no-cover-if-lt-py311 = "sys_version_info < (3, 11) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
+no-cover-if-gte-py311 = "sys_version_info > (3, 11) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
+# Python 3.10
+no-cover-if-is-py310 = "python_version == '3.10' and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
+no-cover-if-lt-py310 = "sys_version_info < (3, 10) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
+no-cover-if-gte-py310 = "sys_version_info > (3, 10) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
+# Python 3.9
+no-cover-if-is-py39 = "python_version == '3.9' and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
+no-cover-if-lt-py39 = "sys_version_info < (3, 9) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
+no-cover-if-gte-py39 = "sys_version_info > (3, 9) and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
+# Python 3.8
+no-cover-if-is-py38 = "python_version == '3.8' and os_environ.get('COVERAGE_EXCLUDE_PYTHON_VERSION') != 'disable'"
+
 [tool.isort]
 profile = "black"
 skip_glob = [

setup.cfg

@@ -64,9 +64,7 @@ install_requires =
     # the most recent version.
     importlib_metadata >= 4.4; python_version <= "3.9"
     packaging >= 22.0
-    # pip 23.1 has an issue with --user installs.
-    # See https://github.com/pypa/pip/issues/11982 for details
-    pip >= 22, != 23.1
+    pip >= 23.1.1
     setuptools >= 60
     wheel >= 0.37
     build >= 0.10
@@ -91,6 +89,7 @@ install_requires =
 # ensure environment consistency.
 dev =
     coverage[toml] == 7.2.5
+    coverage-conditional-plugin == 0.8.0
     pre-commit == 3.2.2
     pytest == 7.3.1
     pytest-xdist == 3.2.1

src/briefcase/__init__.py

@@ -10,7 +10,7 @@
     # Excluded from coverage because a pure test environment (such as the one
     # used by tox in CI) won't have setuptools_scm
     __version__ = get_version("../..", relative_to=__file__)  # pragma: no cover
-except (ModuleNotFoundError, LookupError):
+except (ModuleNotFoundError, LookupError):  # pragma: no-cover-if-missing-setuptools_scm
     # If setuptools_scm isn't in the environment, the call to import will fail.
     # If it *is* in the environment, but the code isn't a git checkout (e.g.,
     # it's been pip installed non-editable) the call to get_version() will fail.

src/briefcase/commands/base.py

@@ -17,12 +17,12 @@
 
 try:
     import importlib_metadata
-except ImportError:
+except ImportError:  # pragma: no-cover-if-lt-py310
     import importlib.metadata as importlib_metadata
 
 try:
     import tomllib
-except ModuleNotFoundError:
+except ModuleNotFoundError:  # pragma: no-cover-if-gte-py310
     import tomli as tomllib
 
 from briefcase import __version__
@@ -177,10 +177,10 @@ def validate_data_path(self, data_path):
                         "The path specified by BRIEFCASE_HOME does not exist."
                     )
             except KeyError:
-                if platform.system() == "Darwin":
+                if platform.system() == "Darwin":  # pragma: no-cover-if-not-macos
                     # macOS uses a bundle name, rather than just the app name
                     app_name = "org.beeware.briefcase"
-                else:
+                else:  # pragma: no-cover-if-is-macos
                     app_name = "briefcase"
 
                 data_path = PlatformDirs(
@@ -214,15 +214,15 @@ def validate_data_path(self, data_path):
                 # performed via ``cmd.exe`` in a different process. Once this
                 # directory exists in the "real" %LOCALAPPDATA%, Windows will
                 # allow normal interactions without attempting to sandbox them.
-                if platform.system() == "Windows":
+                if platform.system() == "Windows":  # pragma: no-cover-if-not-windows
                     subprocess.run(
                         ["mkdir", data_path],
                         shell=True,
                         check=True,
                         stdout=subprocess.DEVNULL,
                         stderr=subprocess.DEVNULL,
                     )
-                else:
+                else:  # pragma: no-cover-if-is-windows
                     os.makedirs(data_path, exist_ok=True)
             except (subprocess.CalledProcessError, OSError):
                 raise BriefcaseCommandError(

src/briefcase/commands/open.py

@@ -10,11 +10,11 @@ class OpenCommand(BaseCommand):
     description = "Open an app in the build tool for the target platform."
 
     def _open_app(self, app: BaseConfig):
-        if self.tools.host_os == "Windows":
+        if self.tools.host_os == "Windows":  # pragma: no-cover-if-not-windows
             self.tools.os.startfile(self.project_path(app))
-        elif self.tools.host_os == "Darwin":
+        elif self.tools.host_os == "Darwin":  # pragma: no-cover-if-not-macos
             self.tools.subprocess.Popen(["open", self.project_path(app)])
-        else:
+        else:  # pragma: no-cover-if-not-linux
             self.tools.subprocess.Popen(["xdg-open", self.project_path(app)])
 
     def open_app(self, app: BaseConfig, **options):

src/briefcase/config.py

@@ -6,7 +6,7 @@
 
 try:
     import tomllib
-except ModuleNotFoundError:
+except ModuleNotFoundError:  # pragma: no-cover-if-gte-py310
     import tomli as tomllib
 
 from briefcase.platforms import get_output_formats, get_platforms

src/briefcase/integrations/docker.py

@@ -347,7 +347,7 @@ def prepare(
                         f"Error building Docker container image for {self.app.app_name}."
                     ) from e
 
-    def _dockerize_path(self, arg: str):
+    def _dockerize_path(self, arg: str):  # pragma: no-cover-if-is-windows
         """Relocate any local path into the equivalent location on the docker
         filesystem.
 
@@ -366,13 +366,20 @@ def _dockerize_path(self, arg: str):
 
         return arg
 
-    def _dockerize_args(self, args, interactive=False, mounts=None, env=None, cwd=None):
+    def _dockerize_args(
+        self,
+        args,
+        interactive=False,
+        mounts=None,
+        env=None,
+        cwd=None,
+    ):  # pragma: no-cover-if-is-windows
         """Convert arguments and environment into a Docker-compatible form. Convert an
         argument and environment specification into a form that can be used as arguments
         to invoke Docker. This involves:
 
          * Configuring the Docker invocation to reference the
-           appropriate container image, and clean up afterwards
+           appropriate container image, and clean up afterward
          * Setting volume mounts for the container instance
          * Transforming any references to local paths into Docker path references.