Run tests under the appropriate 'extra' dependencies

.github/workflows/actions/run-tests/action.yml

@@ -7,21 +7,27 @@
 #   - uses: actions/checkout@v2
 #   - uses: ./.github/workflows/actions/run-tests
 #     with:
+#       tox-env: 'spark'
 #       test-kind: 'unit'
 #       test-marker: 'spark and notebooks'
 
 name: 'Run Python tests'
 description: 'Specify parameters to configure test subsets to run and collect test report for.'
 inputs:
+  tox-env:
+    description: "Name of the tox env. EX) cpu|gpu|spark 
+      See tox.ini at root level for more info"
+    required: true
+    # Options are "cpu|gpu|spark|all" and any default tox env. 
+    # For more info on default tox env, see https://tox.readthedocs.io/en/latest/config.html#tox-environments
+    default: 'all'
   test-kind:
-    description:
-      "The kinds of tests to run. EX) unit|integration|smoke
+    description: "The kinds of tests to run. EX) unit|integration|smoke
       This maps to those in the 'tests/<unit|integration|smoke>' folder"
     required: true
     default: 'unit'
   test-marker:
-    description: 
-      "Finer filter for selecting the tests to run with pytest markers.
+    description: "Finer filter for selecting the tests to run with pytest markers.
       See https://docs.pytest.org/en/6.2.x/example/markers.html"
     default: 'not gpu and not notebooks and not spark'
 outputs:
@@ -44,12 +50,12 @@ runs:
     # '-e py' will use the default 'python' executable found in system path
     # for why using tox, see: https://tox.readthedocs.io/en/latest/index.html
     # tox will do:
-    #   1. build and install source distribution (sdist)
-    #   2. run static analysis on the code (not implemented yet)
-    #   3. run all of the specified test environment (i.e. run tests in different py-versions, etc)
+    #   1. create a virtual env
+    #   2. build and install source distribution (sdist)
+    #   3. run the specified tests
     #   4. show test reports
     run: |
-      tox -e py -- tests/${{ inputs.test-kind }} -m '${{ inputs.test-marker }}'
+      tox -ve ${{ inputs.tox-env }} -- tests/${{ inputs.test-kind }} -m '${{ inputs.test-marker }}'
 
   - name: Prepare Code Coverage Report
     id: rename-cov-report

.github/workflows/nightly.yml

@@ -114,6 +114,7 @@ jobs:
     - name: Run ${{ matrix.test-kind }} tests ('${{ matrix.test-marker }}')
       uses: ./.github/workflows/actions/run-tests
       with:
+        tox-env: 'cpu' # run the cpu tests with the 'recommenders[dev,examples]' dependencies
         test-kind: ${{ matrix.test-kind }}
         test-marker: ${{ matrix.test-marker }}
     # Currently GitHub workflow cannot call an action from another action
@@ -167,6 +168,7 @@ jobs:
     - name: Run ${{ matrix.test-kind }} tests ('${{ matrix.test-marker }}')
       uses: ./.github/workflows/actions/run-tests
       with:
+        tox-env: 'spark' # run the gpu tests with the 'recommenders[spark,examples,dev]' dependencies
         test-kind: ${{ matrix.test-kind }}
         test-marker: ${{ matrix.test-marker }}
 
@@ -202,6 +204,7 @@ jobs:
     - name: Run ${{ matrix.test-kind }} tests ('${{ matrix.test-marker }}')
       uses: ./.github/workflows/actions/run-tests
       with:
+        tox-env: 'gpu' # run the gpu tests with the 'recommenders[gpu,examples,dev]' dependencies
         test-kind: ${{ matrix.test-kind }}
         test-marker: ${{ matrix.test-marker }}
 

.github/workflows/pr-gate.yml

@@ -93,6 +93,7 @@ jobs:
     - name: Run ${{ matrix.test-kind }} tests ('${{ matrix.test-marker }}')
       uses: ./.github/workflows/actions/run-tests
       with:
+        tox-env: 'cpu' # run the cpu tests with the 'recommenders[dev,examples]' dependencies
         test-kind: ${{ matrix.test-kind }}
         test-marker: ${{ matrix.test-marker }}
     # Currently GitHub workflow cannot call an action from another action
@@ -138,6 +139,7 @@ jobs:
     - name: Run ${{ matrix.test-kind }} tests ('${{ matrix.test-marker }}')
       uses: ./.github/workflows/actions/run-tests
       with:
+        tox-env: 'spark' # run the spark tests with the 'recommenders[spark,examples,dev]' dependencies
         test-kind: ${{ matrix.test-kind }}
         test-marker: ${{ matrix.test-marker }}
 
@@ -173,6 +175,7 @@ jobs:
     - name: Run ${{ matrix.test-kind }} tests ('${{ matrix.test-marker }}')
       uses: ./.github/workflows/actions/run-tests
       with:
+        tox-env: 'gpu' # run the gpu tests with the 'recommenders[gpu,examples,dev]' dependencies
         test-kind: ${{ matrix.test-kind }}
         test-marker: ${{ matrix.test-marker }}
 

.gitignore

@@ -87,8 +87,8 @@ celerybeat-schedule
 *.sage.py
 
 # Environments
-.env
-.venv
+.env*
+.venv*
 env/
 venv/
 ENV/

tests/README.md

@@ -157,6 +157,28 @@ Example:
 
 </details>
 
+### Test execution with tox
+
+[Tox](https://tox.readthedocs.io/en/latest/) is a great tool for both virtual environment management and test execution. Tox acts like a front-end for our CI workflows. Our existing [CI pipelines](https://github.com/microsoft/recommenders/actions) in GitHub is leveraging it to orchestrate the build. This way we can provide a **parity** in both local and remote execution environments if both run tox. Run tox and no more **"tests run fine in my dev box but fail in the remote build"**! 
+
+1. If you haven't, `pip install tox`
+2. To run static analysis: `tox -e flake8`
+3. To run any of our test suites:
+    `tox -e {TOX_ENV} -- {PYTEST_PARAM}`
+
+    where 
+    - `TOX_ENV` can be `cpu|gpu|spark|all`, each env maps to the "extra" dependency, for example recommenders[gpu], and recommenders[spark]. It can also be any of the [default envs](https://tox.readthedocs.io/en/latest/config.html#tox-environments): `py|pyNM`
+    - `PYTEST_PARAM` are any standard parameters to supply to `pytest` cli.
+
+    For example:
+
+    1. `tox -e cpu -- tests/unit -m "not notebook and not spark and not gpu` (runs the unit tests with `recommenders[dev,example]` dependencies)
+    2. `tox -e gpu -- tests/unit -m "gpu and notebook"` (runs the gpu notebook tests with `recommenders[dev,example,gpu]` dependencies)
+    3. `tox -e spark -- tests/unit -m "spark and notebook"` (runs the spark notebook tests with `recommenders[dev,example,spark]` dependencies)
+    4. `tox -e all -- tests/unit` (to run all of the unit tests with `recommenders[all]` dependencies)
+    5. `tox -e py -- tests/unit` (runs the unit tests under the default python interpreter with `recommenders[all]`)
+    6. `tox -e py37 -- tests/unit` (runs the unit tests under Python3.7 with `recommenders[all]` )
+
 ## How to create tests on notebooks with Papermill
 
 In the notebooks of this repo, we use [Papermill](https://github.com/nteract/papermill) in unit, smoke and integration tests. Papermill is a tool that enables you to parameterize notebooks, execute and collect metrics across the notebooks, and summarize collections of notebooks.

tox.ini

@@ -2,13 +2,11 @@
 # py will use whatever the basepython `python` maps to from PATH
 # you can use py38, for example, to chosse a different version
 # See https://tox.readthedocs.io/en/latest/config.html#tox-environments
-envlist = py
+envlist = py, cpu, gpu, spark, all
 
 
+# Default env settings
 [testenv]
-# Reading additional dependencies to run the test
-# https://tox.readthedocs.io/en/latest/example/basic.html#depending-on-requirements-txt-or-defining-constraints
-; deps = -rrequirements-dev.txt
 # similar to 'pip install recommenders-*.whl[test]'
 extras = all
 commands = 
@@ -20,6 +18,29 @@ commands =
     # See https://tox.readthedocs.io/en/latest/example/general.html for more details
     pytest {posargs}
 
+[testenv:cpu]
+# i.e: 'pip install recommenders-*.whl[dev]'
+# with this dependency subset, we should be able to run the test markers:
+# 1. "not notebooks and not spark and not gpu" (tests for general sdk utilities)
+# 2. "notebooks and not spark and not gpu" (tests for notebook example without extra dependencies)
+extras = dev,examples
+
+[testenv:gpu]
+# with this dependency subset, we should be able to run the test markers:
+# 1. "gpu and not notebook and not spark" (tests for gpu utilities)
+# 2. "gpu and notebooks and not spark" (tests for notebooks needing gpu resources)
+extras = dev,gpu,examples
+
+[testenv:spark]
+# with this dependency subset, we should be able to run the test markers:
+# 1. "spark and not notebook and not spark" (test for spark utilities)
+# 2. "spark and notebooks and not spark" (tests for notebook using spark)
+extras = dev,spark,examples
+
+[testenv:all]
+# i.e: 'pip install recommenders-*.whl[all]'
+# with this, we should be able to run ANY tests
+extras = all
 
 [testenv:flake8]
 deps = flake8
@@ -54,10 +75,11 @@ addopts =
     --cov-append --cov=recommenders --cov-report=term-missing --cov-report=xml --junitxml=junit/test-results.xml
 
 
+[coverage:report]
+skip_empty = true
+
+
 [flake8]
-; # Configs for flake8-import-order, see https://pypi.org/project/flake8-import-order/ for more info.
-; import-order-style=edited
-; application-import-names=recommenders, tests
 # Native flake8 configs
 max-line-length = 140
 exclude =