Merge pull request #17 from scipp/remove-dask-dependency

Remove dask dependency and add new test run with dask
scipp · Jul 25, 2023 · b3e0b6f · b3e0b6f
2 parents 21f53cf + deade7b
commit b3e0b6f
Show file tree

Hide file tree

Showing 18 changed files with 103 additions and 60 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -29,17 +29,16 @@ jobs:
     needs: formatting
     strategy:
       matrix:
-
         os: ['ubuntu-20.04']
-        python:
-          - version: '3.8'
-            tox-env: 'py38'
+        tox-env: [py38, dask]
+        include:
+          - python-version: '3.8'
 
     uses: ./.github/workflows/test.yml
     with:
       os-variant: ${{ matrix.os }}
-      python-version: ${{ matrix.python.version }}
-      tox-env: ${{ matrix.python.tox-env }}
+      python-version: ${{ matrix.python-version }}
+      tox-env: ${{ matrix.tox-env }}
 
   docs:
     needs: tests

diff --git a/conda/meta.yaml b/conda/meta.yaml
@@ -11,8 +11,7 @@ requirements:
     - setuptools
     - setuptools_scm
   run:
-    - dask
-    - injector
+    - graphlib-backport
     - python>=3.8
 
 test:

diff --git a/pyproject.toml b/pyproject.toml
@@ -27,7 +27,7 @@ classifiers = [
 ]
 requires-python = ">=3.8"
 dependencies = [
-  "dask",
+  "graphlib-backport",
 ]
 dynamic = ["version"]
 

diff --git a/requirements/base.in b/requirements/base.in
@@ -1,2 +1 @@
-dask
 graphlib-backport
diff --git a/requirements/base.txt b/requirements/base.txt
@@ -1,33 +1,9 @@
-# SHA1:073780bdf6d10a4be77389043b3efd8683477e8a
+# SHA1:012661df49149d4c83bf7dbaaf75fac941dcaa91
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
 #
 #    pip-compile-multi
 #
-click==8.1.4
-    # via dask
-cloudpickle==2.2.1
-    # via dask
-dask==2023.5.0
-    # via -r base.in
-fsspec==2023.6.0
-    # via dask
 graphlib-backport==1.0.3
     # via -r base.in
-importlib-metadata==6.8.0
-    # via dask
-locket==1.0.0
-    # via partd
-packaging==23.1
-    # via dask
-partd==1.4.0
-    # via dask
-pyyaml==6.0
-    # via dask
-toolz==0.12.0
-    # via
-    #   dask
-    #   partd
-zipp==3.16.0
-    # via importlib-metadata
diff --git a/requirements/ci.txt b/requirements/ci.txt
@@ -15,7 +15,7 @@ charset-normalizer==3.2.0
     # via requests
 colorama==0.4.6
     # via tox
-distlib==0.3.6
+distlib==0.3.7
     # via virtualenv
 filelock==3.12.2
     # via
@@ -32,7 +32,7 @@ packaging==23.1
     #   -r ci.in
     #   pyproject-api
     #   tox
-platformdirs==3.8.1
+platformdirs==3.9.1
     # via
     #   tox
     #   virtualenv
@@ -52,5 +52,5 @@ tox==4.6.4
     # via -r ci.in
 urllib3==2.0.3
     # via requests
-virtualenv==20.23.1
+virtualenv==20.24.0
     # via tox
diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -23,6 +23,10 @@ async-lru==2.0.3
     # via jupyterlab
 cffi==1.15.1
     # via argon2-cffi-bindings
+click==8.1.5
+    # via
+    #   pip-compile-multi
+    #   pip-tools
 fqdn==1.5.1
     # via jsonschema
 isoduration==20.11.0
@@ -31,7 +35,7 @@ json5==0.9.14
     # via jupyterlab-server
 jsonpointer==2.4
     # via jsonschema
-jsonschema[format-nongpl]==4.18.0
+jsonschema[format-nongpl]==4.18.3
     # via
     #   jupyter-events
     #   jupyterlab-server
@@ -48,7 +52,7 @@ jupyter-server==2.7.0
     #   notebook-shim
 jupyter-server-terminals==0.4.4
     # via jupyter-server
-jupyterlab==4.0.2
+jupyterlab==4.0.3
     # via -r dev.in
 jupyterlab-server==2.23.0
     # via jupyterlab
@@ -58,7 +62,7 @@ overrides==7.3.1
     # via jupyter-server
 pip-compile-multi==2.6.3
     # via -r dev.in
-pip-tools==6.14.0
+pip-tools==7.0.0
     # via pip-compile-multi
 prometheus-client==0.17.1
     # via jupyter-server

diff --git a/requirements/docs.txt b/requirements/docs.txt
@@ -56,6 +56,11 @@ idna==3.4
     # via requests
 imagesize==1.4.1
     # via sphinx
+importlib-metadata==6.8.0
+    # via
+    #   jupyter-client
+    #   nbconvert
+    #   sphinx
 importlib-resources==6.0.0
     # via
     #   jsonschema
@@ -74,7 +79,7 @@ jinja2==3.1.2
     #   nbconvert
     #   nbsphinx
     #   sphinx
-jsonschema==4.18.0
+jsonschema==4.18.3
     # via nbformat
 jsonschema-specifications==2023.6.1
     # via jsonschema
@@ -113,7 +118,7 @@ myst-parser==2.0.0
     # via -r docs.in
 nbclient==0.8.0
     # via nbconvert
-nbconvert==7.6.0
+nbconvert==7.7.1
     # via nbsphinx
 nbformat==5.9.1
     # via
@@ -124,6 +129,12 @@ nbsphinx==0.9.2
     # via -r docs.in
 nest-asyncio==1.5.6
     # via ipykernel
+packaging==23.1
+    # via
+    #   ipykernel
+    #   nbconvert
+    #   pydata-sphinx-theme
+    #   sphinx
 pandocfilters==1.5.0
     # via nbconvert
 parso==0.8.3
@@ -134,7 +145,7 @@ pickleshare==0.7.5
     # via ipython
 pkgutil-resolve-name==1.3.10
     # via jsonschema
-platformdirs==3.8.1
+platformdirs==3.9.1
     # via jupyter-core
 prompt-toolkit==3.0.39
     # via ipython
@@ -159,6 +170,8 @@ python-dateutil==2.8.2
     # via jupyter-client
 pytz==2023.3
     # via babel
+pyyaml==6.0.1
+    # via myst-parser
 pyzmq==25.1.0
     # via
     #   ipykernel
@@ -169,7 +182,7 @@ referencing==0.29.1
     #   jsonschema-specifications
 requests==2.31.0
     # via sphinx
-rpds-py==0.8.10
+rpds-py==0.8.11
     # via
     #   jsonschema
     #   referencing
@@ -243,3 +256,7 @@ webencodings==0.5.1
     # via
     #   bleach
     #   tinycss2
+zipp==3.16.2
+    # via
+    #   importlib-metadata
+    #   importlib-resources
diff --git a/requirements/static.txt b/requirements/static.txt
@@ -7,21 +7,21 @@
 #
 cfgv==3.3.1
     # via pre-commit
-distlib==0.3.6
+distlib==0.3.7
     # via virtualenv
 filelock==3.12.2
     # via virtualenv
 identify==2.5.24
     # via pre-commit
 nodeenv==1.8.0
     # via pre-commit
-platformdirs==3.8.1
+platformdirs==3.9.1
     # via virtualenv
 pre-commit==3.3.3
     # via -r static.in
-pyyaml==6.0
+pyyaml==6.0.1
     # via pre-commit
-virtualenv==20.23.1
+virtualenv==20.24.0
     # via pre-commit
 
 # The following packages are considered to be unsafe in a requirements file:

diff --git a/requirements/test-dask.in b/requirements/test-dask.in
@@ -0,0 +1,2 @@
+-r test.in
+dask
diff --git a/requirements/test-dask.txt b/requirements/test-dask.txt
@@ -0,0 +1,30 @@
+# SHA1:221893b0ff8260477cf0422e864992d973ba8abe
+#
+# This file is autogenerated by pip-compile-multi
+# To update, run:
+#
+#    pip-compile-multi
+#
+-r test.txt
+click==8.1.5
+    # via dask
+cloudpickle==2.2.1
+    # via dask
+dask==2023.5.0
+    # via -r test-dask.in
+fsspec==2023.6.0
+    # via dask
+importlib-metadata==6.8.0
+    # via dask
+locket==1.0.0
+    # via partd
+partd==1.4.0
+    # via dask
+pyyaml==6.0.1
+    # via dask
+toolz==0.12.0
+    # via
+    #   dask
+    #   partd
+zipp==3.16.2
+    # via importlib-metadata
diff --git a/requirements/test.txt b/requirements/test.txt
@@ -12,6 +12,8 @@ iniconfig==2.0.0
     # via pytest
 numpy==1.24.4
     # via -r test.in
+packaging==23.1
+    # via pytest
 pluggy==1.2.0
     # via pytest
 pytest==7.4.0

diff --git a/src/sciline/pipeline.py b/src/sciline/pipeline.py
@@ -18,7 +18,7 @@
     overload,
 )
 
-from .scheduler import DaskScheduler, Scheduler
+from .scheduler import DaskScheduler, NaiveScheduler, Scheduler
 
 T = TypeVar('T')
 
@@ -239,6 +239,11 @@ def __init__(
     ) -> None:
         self._graph = graph
         self._keys = keys
+        if scheduler is None:
+            try:
+                scheduler = DaskScheduler()
+            except ImportError:
+                scheduler = NaiveScheduler()
         self._scheduler = scheduler or DaskScheduler()
 
     def compute(self, keys: Optional[type | Tuple[type, ...]] = None) -> Any:

diff --git a/src/sciline/scheduler.py b/src/sciline/scheduler.py
@@ -51,7 +51,7 @@ def __init__(self, scheduler: Optional[Callable[..., Any]] = None) -> None:
         if scheduler is None:
             import dask
 
-            self._dask_get = dask.get  # type: ignore[attr-defined]
+            self._dask_get = dask.threaded.get  # type: ignore[attr-defined]
         else:
             self._dask_get = scheduler
 

diff --git a/tests/complex_workflow_test.py b/tests/complex_workflow_test.py
@@ -3,15 +3,11 @@
 from dataclasses import dataclass
 from typing import NewType, TypeVar
 
-import dask
 import numpy as np
 import numpy.typing as npt
 
 import sciline as sl
 
-# We use dask with a single thread, to ensure that call counting below is correct.
-dask.config.set(scheduler='synchronous')
-
 
 @dataclass
 class RawData:

diff --git a/tests/dask_scheduler_test.py b/tests/dask_scheduler_test.py
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2023 Scipp contributors (https://github.com/scipp)
+import pytest
+
+import sciline as sl
+
+dask = pytest.importorskip("dask")
+
+
+def test_dask_scheduler_is_threaded_by_default() -> None:
+    scheduler = sl.scheduler.DaskScheduler()
+    assert scheduler._dask_get == dask.threaded.get
diff --git a/tests/pipeline_test.py b/tests/pipeline_test.py
@@ -4,14 +4,10 @@
 from graphlib import CycleError
 from typing import Generic, List, NewType, TypeVar
 
-import dask
 import pytest
 
 import sciline as sl
 
-# We use dask with a single thread, to ensure that call counting below is correct.
-dask.config.set(scheduler='synchronous')
-
 
 def int_to_float(x: int) -> float:
     return 0.5 * x

diff --git a/tox.ini b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py38
+envlist = py38,dask
 isolated_build = true
 
 [testenv]
@@ -8,6 +8,12 @@ setenv =
   JUPYTER_PLATFORM_DIRS = 1
 commands = pytest
 
+[testenv:dask]
+deps = -r requirements/test-dask.txt
+setenv =
+  JUPYTER_PLATFORM_DIRS = 1
+commands = pytest
+
 [testenv:docs]
 description = invoke sphinx-build to build the HTML docs
 deps =