[CI] Run tests against pre-built/installed version of setuptools

.codecov.yml

@@ -2,4 +2,7 @@ comment: false
 coverage:
   status:
     project:
-      threshold: 0.5%
+      default:
+        threshold: 0.5%
+fixes:
+  - "src/::"        # reduce root e.g., "src/setuptools/" => "setuptools/"

.coveragerc

@@ -1,7 +1,31 @@
 [run]
+source_pkgs =
+	setuptools
+	pkg_resources
 omit =
 	# leading `*/` for pytest-dev/pytest-cov#456
-	*/.tox/*
+	*/_vendor/*
+	*/_distutils/*
+	*/tests/*
+	*/test_*/*
+	*/pip-run-*/*
+	*/pip-build*/*
+	*/pip-req-build*/*
+	*/pip-standalone*/*
+	/tmp/easy_install*/*
+	/tmp/tmp*/setup.py
+
+[paths]
+setuptools =
+	setuptools/
+	*/site-packages/setuptools/
+	*/pip-install-*/setuptools_*/setuptools/
+	*/tmp*/setuptools
+pkg_resources =
+	pkg_resources/
+	*/site-packages/pkg_resources/
+	*/pip-install-*/setuptools_*/pkg_resources/
+	*/tmp*/pkg_resources
 
 [report]
 show_missing = True

.coveragerc.ci

@@ -0,0 +1,25 @@
+[run]
+source_pkgs =
+	setuptools
+	pkg_resources
+omit =
+	# leading `*/` for pytest-dev/pytest-cov#456
+	*/_vendor/*
+	*/_distutils/*
+	*/tests/*
+	*/test_*
+
+[paths]
+setuptools =
+	src/setuptools/
+	*/site-packages/setuptools/
+	*/pip-install-*/setuptools_*/setuptools/
+	*/tmp*/setuptools
+pkg_resources =
+	src/pkg_resources/
+	*/site-packages/pkg_resources/
+	*/pip-install-*/setuptools_*/pkg_resources/
+	*/tmp*/pkg_resources
+
+[report]
+show_missing = True

.github/workflows/main.yml

@@ -3,8 +3,74 @@ name: tests
 on: [push, pull_request, workflow_dispatch]
 
 jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      python-version-for-build: ${{ steps.python-version-for-build.outputs.version }}
+      wheel-distribution: ${{ steps.wheel-distribution.outputs.path }}
+      sdist-distribution: ${{ steps.sdist-distribution.outputs.path }}
+
+    steps:
+      - name: Set Python version for build step
+        id: python-version-for-build
+        run: echo "::set-output name=version::3.10"  # <- Python version used for build release
+      - uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ steps.python-version-for-build.outputs.version }}
+      - name: Install tox
+        run: python -m pip install tox
+      - name: Lint code
+        run: python -m tox -e lint
+      - name: Build distribution files
+        run: python -m tox -e build
+      - name: Record the path of wheel distribution
+        id: wheel-distribution
+        run: |
+          echo "::set-output name=path::$(ls dist/*.whl)"
+      - name: Record the path of sdist distribution
+        id: sdist-distribution
+        run: |
+          echo "::set-output name=path::$(ls dist/*.tar.gz)"
+      - name: Store the distribution files for use in other stages
+        uses: actions/upload-artifact@v2
+        with:
+          name: python-distribution-files
+          path: |
+            ${{ steps.wheel-distribution.outputs.path }}
+            ${{ steps.sdist-distribution.outputs.path }}
+          retention-days: 2
+      - name: Debug pipeline stage output
+        run: |
+          echo python-version-for-build = ${{ steps.python-version-for-build.outputs.version }}
+          echo wheel-distribution = ${{ steps.wheel-distribution.outputs.path }}
+          echo sdist-distribution = ${{ steps.sdist-distribution.outputs.path }}
+
+  develop-test:
+    # Make sure `setuptools` can be installed in editable mode
+    # and tests also run as they would do in the developer's machine
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          # Use a release that is not very new but still have a long life:
+          python-version: "3.8"
+      - name: Ensure setuptools can be installed in editable mode
+        run: |
+          python -m pip install -e .
+      - name: Install tox
+        run: |
+          python -m pip install tox
+      - name: Run tests
+        run: python -m tox -- --cov-report xml --cov-report term
+
   test:
+    needs: prepare
     strategy:
+      fail-fast: false
       matrix:
         distutils:
         - stdlib
@@ -22,6 +88,8 @@ jobs:
     runs-on: ${{ matrix.platform }}
     env:
       SETUPTOOLS_USE_DISTUTILS: ${{ matrix.distutils }}
+      PRE_BUILT_SETUPTOOLS_WHEEL: ${{ needs.prepare.outputs.wheel-distribution }}
+      PRE_BUILT_SETUPTOOLS_SDIST: ${{ needs.prepare.outputs.sdist-distribution }}
     steps:
       - uses: actions/checkout@v2
       - name: Setup Python
@@ -31,17 +99,44 @@ jobs:
       - name: Install tox
         run: |
           python -m pip install tox
+      - name: Isolate tests to avoid PYTHONPATH interference
+        run: |
+          # Avoid problems with import paths, see #2318, #3015:
+          # `mkdir` `git rm` and `git mv` are OS-independent commands
+          git mv setuptools/tests test_setuptools
+          git mv pkg_resources/tests test_pkg_resources
+          # Prevent local directories to take precedence over installed packages:
+          mkdir src
+          git mv _distutils_hack src
+          git mv setuptools src
+          git mv pkg_resources src
+      - name: Retrieve pre-built distribution files
+        uses: actions/download-artifact@v2
+        with:
+          name: python-distribution-files
+          path: dist/
+      - name: Skip some Pytest plugins on PyPy
+        if: contains(matrix.python, 'pypy') && matrix.distutils == 'local' && matrix.platform != 'windows-latest'
+        # PyPy 3.7 imports `distutils` in its `sysconfig`, which would trigger
+        # `_distutils_hack` and potentially mess imports for `pep517` (used by some plugins)
+        # https://foss.heptapod.net/pypy/pypy/-/blob/8301595a71447bbe58bb61c5ecd83d93cfda5eb0/lib-python/3/sysconfig.py#L461-473
+        # https://foss.heptapod.net/pypy/pypy/-/blob/8301595a71447bbe58bb61c5ecd83d93cfda5eb0/lib_pypy/_sysconfigdata.py#L5
+        # https://github.com/pypa/pep517/blob/a942316305395f8f757f210e2b16f738af73f8b8/pep517/in_process/_in_process.py#L76-L86
+        run: echo 'PYTEST_ADDOPTS=-p no:perf' >> $GITHUB_ENV
       - name: Run tests
-        run: tox -- --cov-report xml
+        run: >-
+          python -m tox
+          --installpkg '${{ needs.prepare.outputs.wheel-distribution }}'
+          -- --cov-report xml --cov-report term --cov-config .coveragerc.ci
       - name: Publish coverage
-        if: false  # disabled for #2727
         uses: codecov/codecov-action@v1
         with:
           flags: >-  # Mark which lines are covered by which envs
             ${{ runner.os }},
             ${{ matrix.python }}
 
   test_cygwin:
+    needs: prepare
     strategy:
       matrix:
         distutils:
@@ -50,6 +145,8 @@ jobs:
     runs-on: windows-latest
     env:
       SETUPTOOLS_USE_DISTUTILS: ${{ matrix.distutils }}
+      PRE_BUILT_SETUPTOOLS_WHEEL: ${{ needs.prepare.outputs.wheel-distribution }}
+      PRE_BUILT_SETUPTOOLS_SDIST: ${{ needs.prepare.outputs.sdist-distribution }}
     steps:
       - uses: actions/checkout@v2
       - name: Install Cygwin with Python
@@ -66,18 +163,43 @@ jobs:
         shell: C:\cygwin\bin\env.exe CYGWIN_NOWINPATH=1 CHERE_INVOKING=1 C:\cygwin\bin\bash.exe -leo pipefail -o igncr {0}
         run: |
           python3.8 -m pip install tox
-      - name: Run tests
+      - name: Isolate tests to avoid PYTHONPATH interference
         shell: C:\cygwin\bin\env.exe CYGWIN_NOWINPATH=1 CHERE_INVOKING=1 C:\cygwin\bin\bash.exe -leo pipefail -o igncr {0}
         run: |
-          tox -- --cov-report xml
+          # Avoid problems with import paths, see #2318, #3015:
+          # `mkdir` `git rm` and `git mv` are OS-independent commands
+          git mv setuptools/tests test_setuptools
+          git mv pkg_resources/tests test_pkg_resources
+          # Prevent local directories to take precedence over installed packages:
+          mkdir src
+          git mv _distutils_hack src
+          git mv setuptools src
+          git mv pkg_resources src
+      - name: Retrieve pre-built distribution files
+        uses: actions/download-artifact@v2
+        with:
+          name: python-distribution-files
+          path: dist/
+      - name: Run tests
+        shell: C:\cygwin\bin\env.exe CYGWIN_NOWINPATH=1 CHERE_INVOKING=1 C:\cygwin\bin\bash.exe -leo pipefail -o igncr {0}
+        run: >-
+          python -m tox
+          --installpkg '${{ needs.prepare.outputs.wheel-distribution }}'
+          -- --cov-report xml --cov-report term --cov-config .coveragerc.ci
+      - name: Publish coverage
+        uses: codecov/codecov-action@v1
+        with:
+          flags: >-  # Mark which lines are covered by which envs
+            cygwin
+            ${{ matrix.python }}
 
   integration-test:
     strategy:
       matrix:
         distutils:
         - stdlib
         - local
-    needs: test
+    needs: [prepare, test]
     if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && contains(github.ref, 'refs/tags/'))
     # To avoid long times and high resource usage, we assume that:
     # 1. The setuptools APIs used by packages don't vary too much with OS or
@@ -89,6 +211,8 @@ jobs:
     runs-on: ubuntu-latest
     env:
       SETUPTOOLS_USE_DISTUTILS: ${{ matrix.distutils }}
+      PRE_BUILT_SETUPTOOLS_WHEEL: ${{ needs.prepare.outputs.wheel-distribution }}
+      PRE_BUILT_SETUPTOOLS_SDIST: ${{ needs.prepare.outputs.sdist-distribution }}
     steps:
       - uses: actions/checkout@v2
       - name: Install OS-level dependencies
@@ -103,11 +227,31 @@ jobs:
       - name: Install tox
         run: |
           python -m pip install tox
+      - name: Isolate tests to avoid PYTHONPATH interference
+        run: |
+          # Avoid problems with import paths, see #2318, #3015:
+          # `mkdir` `git rm` and `git mv` are OS-independent commands
+          git mv setuptools/tests test_setuptools
+          git mv pkg_resources/tests test_pkg_resources
+          # Prevent local directories to take precedence over installed packages:
+          mkdir src
+          git mv _distutils_hack src
+          git mv setuptools src
+          git mv pkg_resources src
+      - name: Retrieve pre-built distribution files
+        uses: actions/download-artifact@v2
+        with:
+          name: python-distribution-files
+          path: dist/
       - name: Run integration tests
-        run: tox -e integration
+        run: >-
+          python -m tox -e integration
+          --installpkg '${{ needs.prepare.outputs.wheel-distribution }}'
+          -- -vv --durations=10
+          test_setuptools/integration
 
   release:
-    needs: [test, test_cygwin, integration-test]
+    needs: [prepare, test, test_cygwin, develop-test, integration-test]
     if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
 
@@ -116,7 +260,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: "3.10"
+          python-version: ${{ needs.prepare.outputs-python-version-for-build }}
       - name: Install tox
         run: |
           python -m pip install tox

conftest.py

@@ -1,11 +1,9 @@
 import sys
+import os
 
 import pytest
 
 
-pytest_plugins = 'setuptools.tests.fixtures'
-
-
 def pytest_addoption(parser):
     parser.addoption(
         "--package_name", action="append", default=[],
@@ -19,11 +17,13 @@ def pytest_addoption(parser):
 
 def pytest_configure(config):
     config.addinivalue_line("markers", "integration: integration tests")
+    if os.environ.get("PRE_BUILT_SETUPTOOLS_WHEEL"):
+        config.option.perf_target = os.environ["PRE_BUILT_SETUPTOOLS_WHEEL"]
 
 
 collect_ignore = [
+    'src',
     'tests/manual_test.py',
-    'setuptools/tests/mod_with_constant.py',
     'setuptools/_distutils',
     '_distutils_hack',
     'setuptools/extern',
@@ -33,6 +33,10 @@ def pytest_configure(config):
     'pkg_resources/_vendor',
 ]
 
+collect_ignore_glob = [
+    '*/mod_with_constant.py',
+]
+
 
 if sys.version_info < (3, 6):
     collect_ignore.append('docs/conf.py')  # uses f-strings

pkg_resources/tests/test_pkg_resources.py

@@ -175,8 +175,10 @@ def test_setuptools_not_imported(self):
         that action doesn't cause setuptools to be imported.
         """
         lines = (
-            'import pkg_resources',
             'import sys',
+            'sys.modules.pop("setuptools", None)',
+            # ^-- make sure test does not fail if imported elsewhere
+            'import pkg_resources',
             (
                 'assert "setuptools" not in sys.modules, '
                 '"setuptools was imported"'

pkg_resources/tests/test_resources.py

@@ -391,11 +391,11 @@ def test_resolve_conflicts_with_prior(self):
 class TestEntryPoints:
     def assertfields(self, ep):
         assert ep.name == "foo"
-        assert ep.module_name == "pkg_resources.tests.test_resources"
+        assert ep.module_name == __name__
         assert ep.attrs == ("TestEntryPoints",)
         assert ep.extras == ("x",)
         assert ep.load() is TestEntryPoints
-        expect = "foo = pkg_resources.tests.test_resources:TestEntryPoints [x]"
+        expect = f"foo = {__name__}:TestEntryPoints [x]"
         assert str(ep) == expect
 
     def setup_method(self, method):
@@ -404,13 +404,13 @@ def setup_method(self, method):
 
     def testBasics(self):
         ep = EntryPoint(
-            "foo", "pkg_resources.tests.test_resources", ["TestEntryPoints"],
+            "foo", __name__, ["TestEntryPoints"],
             ["x"], self.dist
         )
         self.assertfields(ep)
 
     def testParse(self):
-        s = "foo = pkg_resources.tests.test_resources:TestEntryPoints [x]"
+        s = f"foo = {__name__}:TestEntryPoints [x]"
         ep = EntryPoint.parse(s, self.dist)
         self.assertfields(ep)
 
@@ -489,10 +489,7 @@ def testParseMap(self):
             EntryPoint.parse_map(self.submap_str)
 
     def testDeprecationWarnings(self):
-        ep = EntryPoint(
-            "foo", "pkg_resources.tests.test_resources", ["TestEntryPoints"],
-            ["x"]
-        )
+        ep = EntryPoint("foo", __name__, ["TestEntryPoints"], ["x"])
         with pytest.warns(pkg_resources.PkgResourcesDeprecationWarning):
             ep.load(require=False)