poetry related changes

.github/workflows/validate.yaml

@@ -8,8 +8,6 @@ on:
 
 env:
   FORCE_COLOR: 1
-  XDG_CACHE_HOME: ${{ github.workspace }}/cache
-  PIP_CACHE_DIR: ${{ github.workspace }}/pip-cache
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -53,21 +51,14 @@ jobs:
         with:
           distribution: "temurin"
           java-version: "17"
-      - name: Cache pip
+      - name: Cache XDG_CACHE_HOME
         uses: actions/cache@v3
         with:
-          path: ${{ env.PIP_CACHE_DIR }}
-          key: ${{ matrix.os }}-pip-${{ matrix.python-version }}-v1-${{
-            hashFiles('**/setup.py', '**/*requirements*.txt') }}
+          path: ~/.cache
+          key: ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml', '**/poetry.lock', '**/with-fuseki.sh', '**/*requirements*.txt') }}
           restore-keys: |
-            ${{ matrix.os }}-pip-${{ matrix.python-version }}-v1-
-      - name: Cache xdg
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.XDG_CACHE_HOME }}
-          key: ${{ matrix.os }}-xdg-v1-${{ hashFiles('**/with-fuseki.sh') }}
-          restore-keys: |
-            ${{ matrix.os }}-xdg-v1-
+            ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-
+            ${{ github.job }}-xdg-v1-${{ matrix.os }}-
       - name: Install Task
         uses: arduino/setup-task@v1
         with:
@@ -96,6 +87,8 @@ jobs:
           name: ${{ matrix.python-version }}-${{ matrix.os }}-pytest-junit-xml
           path: test_reports/${{ matrix.python-version }}-${{ matrix.os }}-pytest-junit.xml
   extra-tasks:
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -109,14 +102,16 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Cache pip
+      - name: Python Poetry Action
+        uses: snok/install-poetry@v1
+      - name: Cache XDG_CACHE_HOME
         uses: actions/cache@v3
         with:
-          path: ${{ env.PIP_CACHE_DIR }}
-          key: tox-${{ matrix.task }}-pip-v1-${{
-            hashFiles('**/setup.py', '**/requirements*.txt') }}
+          path: ~/.cache
+          key: ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml', '**/poetry.lock', '**/with-fuseki.sh', '**/*requirements*.txt') }}
           restore-keys: |
-            tox-${{ matrix.task }}-pip-v1-
+            ${{ github.job }}-xdg-v1-${{ matrix.os }}-${{ matrix.python-version }}-
+            ${{ github.job }}-xdg-v1-${{ matrix.os }}-
       - name: Install Task
         uses: arduino/setup-task@v1
         with:

Dockerfile.devcontainer

@@ -47,16 +47,3 @@ RUN \
         poetry==1.3.1 \
         && \
     true
-
-COPY pyproject.toml poetry.lock ./
-
-RUN poetry export \
-    -E dev \
-    -E tests \
-    -E docs \
-    --output /var/tmp/requirements-via-poetry-including-all-dev.txt
-RUN \
-    python -m pip install \
-        -r /var/tmp/requirements-via-poetry-including-all-dev.txt \
-        && \
-    true

Taskfile.yml

@@ -6,23 +6,19 @@ version: "3"
 vars:
   TASKFILE_DIR:
     sh: pwd
-  # The path to the virtual environment to use when using a virtual
-  # environment.
-  VENV_PATH: '{{(env "VENV_PATH") | default (osClean (print .TASKFILE_DIR "/.venv"))}}'
-  # Non empty string results in tasks using a virtual environment.
-  WITH_VENV: '{{env "WITH_VENV" | default "false"}}'
-  # The virtual environemtn specific python interpreter to use.
-  VENV_BINPREFIX: '{{if (mustFromJson .WITH_VENV)}}{{if (eq OS "windows")}}{{.VENV_PATH}}/Scripts/{{else}}{{.VENV_PATH}}/bin/{{end}}{{end}}'
-  VENV_PYTHON: '{{if (eq OS "windows")}}{{.VENV_BINPREFIX}}python.exe{{else}}{{.VENV_BINPREFIX}}python{{end}}'
-  # The python interpreter to use.
+  POETRY: poetry
+  # The prefix to use when running dev commands
+  RUN_PREFIX: "{{.POETRY}} run"
+  # The python to use for running in the venv
+  VENV_PYTHON: "{{.RUN_PREFIX}} python"
+    # The python interpreter to use.
   PYTHON: python
-  _PYTHON: "{{if (mustFromJson .WITH_VENV)}}{{.VENV_PYTHON}}{{else}}{{.PYTHON}}{{end}}"
   # Truthish values ("true", "1", etc.) results in java being installed as a
   # system dependency.
   INSTALL_SYSTEM_DEPS_JAVA: "false"
   # Truthish values ("true", "1", etc.) results in extras being installed with
   # pip.
-  INSTALL_PIP_EXTRAS: "false"
+  INSTALL_EXTRAS: "false"
   # Truthish values ("true", "1", etc.) results in extensive tests being ran and
   # dependencies for extensive tests being installed.
   EXTENSIVE: "false"
@@ -61,60 +57,40 @@ tasks:
   install:tox:
     desc: Install tox
     cmds:
-      - "{{._PYTHON | shellQuote}} -m pip install tox {{if (mustFromJson .WITH_GITHUB_ACTIONS)}}tox-gh-actions{{end}}"
+      - "{{.PYTHON}} -m pip install tox {{if (mustFromJson .WITH_GITHUB_ACTIONS)}}tox-gh-actions{{end}}"
 
-  install:pip-deps:
-    desc: Install pip dependencies
+  poetry:configure:
+    desc: Configure the environment for development with poetry
     cmds:
-      - "poetry export -E dev -E tests -E docs {{if (mustFromJson .INSTALL_PIP_EXTRAS)}}-E berkeleydb -E networkx{{end}} --output /var/tmp/requirements-via-poetry-incl-dev.txt"
-      - "{{._PYTHON | shellQuote}} -m pip install --upgrade -r /var/tmp/requirements-via-poetry-incl-dev.txt"
-
-  install:flake8:
-    desc: Install flake8 dependencies
-    cmds:
-      - poetry export --only flake8 --output /var/tmp/requirements.flake8.txt
-      - "{{._PYTHON | shellQuote}} -m pip install --upgrade -r /var/tmp/requirements.flake8.txt"
-
-  install:deps:
-    desc: Install all dependencies
-    cmds:
-      - task: install:system-deps
-      - task: install:tox
-      - task: install:pip-deps
+      - |
+        {{if .POETRY_PYTHON}} {{.POETRY}} env use {{.POETRY_PYTHON}} {{end}}
+        {{.POETRY}} install {{if (or (mustFromJson .INSTALL_EXTRAS) (mustFromJson .EXTENSIVE))}} --all-extras{{end}} {{.CLI_ARGS}}
 
-  venv:create:
-    desc: Create a venv to VENV_PATH(={{.VENV_PATH}})
+  configure:
+    desc: Configure the environment for development
     cmds:
-      - "{{.PYTHON | shellQuote}} -m venv {{.VENV_PATH}}"
+      - task: venv:install
 
   venv:install:
-    desc: Create and install a venv to VENV_PATH(={{.VENV_PATH}})
+    desc: Create and install a venv
     cmds:
-      - task: venv:create
-      - task: install:pip-deps
-        vars: { _PYTHON: "{{.VENV_PYTHON}}" }
+      - task: poetry:configure
 
   venv:clean:
-    desc: Remove the venv at VENV_PATH(={{.VENV_PATH}})
+    desc: Clean the virtual environment
     cmds:
-      - task: _rimraf
-        vars: { _PYTHON: "{{.PYTHON}}", RIMRAF_TARGET: "{{.VENV_PATH}}" }
+      - "{{.POETRY}} env remove --all {{.CLI_ARGS}}"
 
   venv:run:
     desc: Run a command inside the venv
     cmds:
       - cmd: |
-          VIRTUAL_ENV="{{.VENV_PATH}}" PATH="{{.VENV_PATH}}/bin${PATH:+:${PATH}}" {{.CLI_ARGS}}
+          {{.RUN_PREFIX}} {{.CLI_ARGS}}
 
   run:
     desc: Run a command. If WITH_VENV is truthish, the command will be run inside the virtual environment.
     cmds:
-      - cmd: |
-          {{if (mustFromJson .WITH_VENV)}}
-          VIRTUAL_ENV="{{.VENV_PATH}}" PATH="{{.VENV_PATH}}/bin${PATH:+:${PATH}}" {{.CLI_ARGS}}
-          {{else}}
-          {{.CLI_ARGS}}
-          {{end}}
+      - task: venv:run
 
   tox:
     desc: Run tox
@@ -126,38 +102,38 @@ tasks:
         {{if .TOX_JUNIT_XML_PREFIX}}TOX_JUNIT_XML_PREFIX={{shellQuote .TOX_JUNIT_XML_PREFIX}}{{end}} \
         {{if .COVERAGE_FILE}}COVERAGE_FILE={{shellQuote .COVERAGE_FILE}}{{end}} \
         {{.TEST_HARNESS}} \
-        {{._PYTHON | shellQuote}} \
+        {{.PYTHON | shellQuote}} \
           -m tox \
           {{.CLI_ARGS}}
     env:
       TOXENV: '{{if .TOX_PYTHON_VERSION}}py{{mustRegexReplaceAll "^([0-9]+)[.]([0-9]+).*" .TOX_PYTHON_VERSION "${1}${2}"}}{{if (mustFromJson .EXTENSIVE)}}-extensive{{end}}{{.TOXENV_SUFFIX | default ""}}{{end}}'
   test:
     desc: Run tests
     cmds:
-      - '{{.TEST_HARNESS}}{{print .VENV_BINPREFIX "pytest" | shellQuote}} {{if (mustFromJson .WITH_COVERAGE)}}--cov --cov-report={{end}} {{.CLI_ARGS}}'
+      - '{{.TEST_HARNESS}}{{.RUN_PREFIX}} pytest {{if (mustFromJson .WITH_COVERAGE)}}--cov --cov-report={{end}} {{.CLI_ARGS}}'
   flake8:
     desc: Run flake8
     cmds:
       - |
-        if {{._PYTHON | shellQuote}} -c 'import importlib; exit(0 if importlib.util.find_spec("flakeheaven") is not None else 1)'
+        if {{.VENV_PYTHON}} -c 'import importlib.util; exit(0 if importlib.util.find_spec("flakeheaven") is not None else 1)'
         then
           1>&2 echo "running flakeheaven"
-          {{._PYTHON | shellQuote}} -m flakeheaven lint {{.CLI_ARGS}}
+          {{.VENV_PYTHON}} -m flakeheaven lint {{.CLI_ARGS}}
         else
           1>&2 echo "skipping flakeheaven as it is not installed, likely because python version is older than 3.8"
         fi
   black:
     desc: Run black
     cmds:
-      - '{{._PYTHON | shellQuote}} -m black {{if (mustFromJson (.CHECK | default "false"))}}--check --diff {{end}}{{.CLI_ARGS | default "."}}'
+      - '{{.VENV_PYTHON}} -m black {{if (mustFromJson (.CHECK | default "false"))}}--check --diff {{end}}{{.CLI_ARGS | default "."}}'
   isort:
     desc: Run isort
     cmds:
-      - '{{._PYTHON | shellQuote}} -m isort {{if (mustFromJson (.CHECK | default "false"))}}--check --diff {{end}}{{.CLI_ARGS | default "."}}'
+      - '{{.VENV_PYTHON}} -m isort {{if (mustFromJson (.CHECK | default "false"))}}--check --diff {{end}}{{.CLI_ARGS | default "."}}'
   mypy:
     desc: Run mypy
     cmds:
-      - "{{._PYTHON | shellQuote}} -m mypy --show-error-context --show-error-codes {{.CLI_ARGS}}"
+      - "{{.VENV_PYTHON}} -m mypy --show-error-context --show-error-codes {{.CLI_ARGS}}"
 
   lint:fix:
     desc: Fix auto-fixable linting errors
@@ -201,7 +177,7 @@ tasks:
     desc: Build documentation
     cmds:
       - echo "PYTHONPATH=${PYTHONPATH}"
-      - "{{._PYTHON | shellQuote}} -m sphinx.cmd.build -b html -d docs/_build/doctrees docs/ docs/_build/html -W {{.CLI_ARGS}}"
+      - "{{.VENV_PYTHON}} -m sphinx.cmd.build -T -W -b html -d docs/_build/doctree docs docs/_build/html {{.CLI_ARGS}}"
 
   docs:live-server:
     desc: Run a live server on generated docs
@@ -244,7 +220,7 @@ tasks:
   test:data:fetch:
     desc: Fetch test data.
     cmds:
-      - "{{._PYTHON}} test/data/fetcher.py {{.CLI_ARGS}}"
+      - "{{.VENV_PYTHON}} test/data/fetcher.py {{.CLI_ARGS}}"
 
   pre-commit:install:
     desc: Install pre-commit hooks
@@ -272,7 +248,7 @@ tasks:
       - task: install:tox
         vars:
           WITH_GITHUB_ACTIONS: true
-      - cmd: "{{._PYTHON | shellQuote}} -m pip install coveralls"
+      - cmd: "{{.PYTHON}} -m pip install coveralls"
       - task: tox
         vars:
           COVERAGE_FILE: ".coverage"
@@ -281,13 +257,15 @@ tasks:
   gha:flake8:
     desc: GitHub Actions flake8 workflow
     cmds:
-      - task: install:flake8
+      - task: poetry:configure
+        vars:
+          .CLI_ARGS: --no-root --only=flake8
       - task: flake8
 
   cmd:rdfpipe:
     desc: Run rdfpipe
     cmds:
-      - cmd: "{{._PYTHON | shellQuote}} -m rdflib.tools.rdfpipe {{.CLI_ARGS}}"
+      - cmd: "{{.VENV_PYTHON}} -m rdflib.tools.rdfpipe {{.CLI_ARGS}}"
 
   pip-compile:
     cmds:
@@ -356,7 +334,7 @@ tasks:
     # a python interpreter. The name is inspired by
     # <https://www.npmjs.com/package/rimraf>.
     - cmd: |
-        {{._PYTHON | shellQuote}} -c '
+        {{.PYTHON}} -c '
         from pathlib import Path;
         import sys, shutil;
         for path in sys.argv[1:]:

docker-compose.yml

@@ -6,6 +6,8 @@ services:
       context: .
       dockerfile: Dockerfile.devcontainer
     working_dir: /srv/workspace
+    environment:
+      - POETRY_VIRTUALENVS_IN_PROJECT=1
     volumes:
       - .:/srv/workspace:z,cached
       - xdg-cache-home:/root/.cache

docs/conf.py

@@ -262,7 +262,12 @@ def find_version(filename):
     "ref.python",
 ]
 
-nitpicky = True
+if sys.version_info < (3, 8):
+    # Being nitpicky on python 3.7 causes lots of problems with Sphinx 4, so
+    # turn it off there.
+    nitpicky = False
+else:
+    nitpicky = True
 
 nitpick_ignore = [
     ("py:data", "typing.Literal"),
@@ -310,10 +315,3 @@ def find_version(filename):
             ("py:class", "TextIO"),
         ]
     )
-
-if sys.version_info < (3, 8):
-    nitpick_ignore.extend(
-        [
-            ("py:class", "importlib_metadata.EntryPoint"),
-        ]
-    )