From 1ce62bd012ff35bd4a6b7e65fb40527f888a01e7 Mon Sep 17 00:00:00 2001 From: Simon Zhao <43029286+simonzhaoms@users.noreply.github.com> Date: Fri, 18 Nov 2022 10:21:34 +0800 Subject: [PATCH 1/5] Register custom marks to avoid unknown mark warnings --- pyproject.toml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 251c7b5aa8..e5720189b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,3 +10,17 @@ dependencies = [ "numpy>=1.15", ] build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +markers = [ + "deeprec: tests for deeprec model", + "experimental", + "flaky: flaky tests that can fail unexpectedly", + "gpu: tests running on GPU", + "integration: integration tests", + "notebooks: tests for notebooks", + "sequential", + "smoke: smoke tests" + "spark: tests that requires Spark", + "vw: tests for vowpal wabbit", +] \ No newline at end of file From 8678329b37fa3c4cb35a2ca2895228ff5891c069 Mon Sep 17 00:00:00 2001 From: Simon Zhao <43029286+simonzhaoms@users.noreply.github.com> Date: Fri, 18 Nov 2022 10:34:19 +0800 Subject: [PATCH 2/5] Update docs --- pyproject.toml | 2 +- tests/README.md | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e5720189b4..d90f5ce436 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ markers = [ "integration: integration tests", "notebooks: tests for notebooks", "sequential", - "smoke: smoke tests" + "smoke: smoke tests", "spark: tests that requires Spark", "vw: tests for vowpal wabbit", ] \ No newline at end of file diff --git a/tests/README.md b/tests/README.md index 055af454d8..5009a82549 100644 --- a/tests/README.md +++ b/tests/README.md @@ -59,9 +59,14 @@ You want to make sure that all your code works before you submit it to the repos * It is better to create multiple small tests than one large test that checks all the code. * Use `@pytest.fixture` to create data in your tests. -* Use the mark `@pytest.mark.gpu` if you want the test to be executed in a GPU environment. Use `@pytest.mark.spark` if you want the test to be executed in a Spark environment. -* Use `@pytest.mark.smoke` and `@pytest.mark.integration` to mark the tests as smoke tests and integration tests. +* Use the mark `@pytest.mark.gpu` if you want the test to be executed + in a GPU environment. Use `@pytest.mark.spark` if you want the test + to be executed in a Spark environment. +* Use `@pytest.mark.smoke` and `@pytest.mark.integration` to mark the + tests as smoke tests and integration tests. * Use `@pytest.mark.notebooks` if you are testing a notebook. +* Use `@pytest.mark.deeprec` to mark the tests for deeprec model. +* Use `@pytest.mark.vw` to mark the tests for vowpal wabbit. * Avoid using `is` in the asserts, instead use the operator `==`. * Follow the pattern `assert computation == value`, for example: ```python From 2fa0c007b30af3b5686a23c16503cc9b276d703d Mon Sep 17 00:00:00 2001 From: Simon Zhao <43029286+simonzhaoms@users.noreply.github.com> Date: Fri, 18 Nov 2022 10:42:05 +0800 Subject: [PATCH 3/5] Correct links --- tests/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/README.md b/tests/README.md index 5009a82549..cb4c265b09 100644 --- a/tests/README.md +++ b/tests/README.md @@ -38,7 +38,7 @@ In the following figure we show a workflow on how the tests are executed via Azu GitHub workflows `azureml-unit-tests.yml`, `azureml-cpu-nightly.yml`, `azureml-gpu-nightly.yml` and `azureml-spark-nightly` located in [.github/workflows/](../.github/workflows/) are used to run the tests on AzureML. The parameters to configure AzureML are defined in the workflow yml files. The tests are divided into groups and each workflow triggers these test groups in parallel, which significantly reduces end-to-end execution time. -There are three scripts used with each workflow, all of them are located in [test/ci/azureml_tests](./ci/azureml_tests/): +There are three scripts used with each workflow, all of them are located in [ci/azureml_tests](./ci/azureml_tests/): * `submit_groupwise_azureml_pytest.py`: this script uses parameters in the workflow yml to set up the AzureML environment for testing using the AzureML SDK. * `run_groupwise_pytest.py`: this script uses pytest to run the tests of the libraries and notebooks. This script runs in an AzureML workspace with the environment created by the script above. @@ -116,7 +116,7 @@ The way papermill works to inject parameters is very simple, it generates a copy The second modification that we need to do to the notebook is to record the metrics we want to test using `sb.glue("output_variable", python_variable_name)`. We normally use the last cell of the notebook to record all the metrics. These are the metrics that we are going to control in the smoke and integration tests. -This is an example on how we do a smoke test. The complete code can be found in [tests/smoke/examples/test_notebooks_python.py](tests/smoke/examples/test_notebooks_python.py): +This is an example on how we do a smoke test. The complete code can be found in [smoke/examples/test_notebooks_python.py](./smoke/examples/test_notebooks_python.py): ```python import pytest @@ -160,7 +160,7 @@ To add a new test to the AzureML pipeline, add the test path to an appropriate t Tests in `group_cpu_xxx` groups are executed on a CPU-only AzureML compute cluster node. Tests in `group_gpu_xxx` groups are executed on a GPU-enabled AzureML compute cluster node with GPU related dependencies added to the AzureML run environment. Tests in `group_pyspark_xxx` groups are executed on a CPU-only AzureML compute cluster node, with the PySpark related dependencies added to the AzureML run environment. -It's important to keep in mind while adding a new test that the runtime of the test group should not exceed the specified threshold in [test_groups.py](tests/ci/azureml_tests/test_groups.py). +It's important to keep in mind while adding a new test that the runtime of the test group should not exceed the specified threshold in [test_groups.py](./ci/azureml_tests/test_groups.py). Example of adding a new test: From 33c73ff9dfd36d18851f0e098f9adc7e8ce5c287 Mon Sep 17 00:00:00 2001 From: Simon Zhao <43029286+simonzhaoms@users.noreply.github.com> Date: Fri, 18 Nov 2022 13:06:30 +0800 Subject: [PATCH 4/5] Remove marks for deeprec and vw in the doc Co-authored-by: Miguel Fierro <3491412+miguelgfierro@users.noreply.github.com> --- tests/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/README.md b/tests/README.md index cb4c265b09..e94a6640df 100644 --- a/tests/README.md +++ b/tests/README.md @@ -65,8 +65,6 @@ You want to make sure that all your code works before you submit it to the repos * Use `@pytest.mark.smoke` and `@pytest.mark.integration` to mark the tests as smoke tests and integration tests. * Use `@pytest.mark.notebooks` if you are testing a notebook. -* Use `@pytest.mark.deeprec` to mark the tests for deeprec model. -* Use `@pytest.mark.vw` to mark the tests for vowpal wabbit. * Avoid using `is` in the asserts, instead use the operator `==`. * Follow the pattern `assert computation == value`, for example: ```python From 7dcd56b2ea6782e1922125cdc38d00523168755e Mon Sep 17 00:00:00 2001 From: Simon Zhao <43029286+simonzhaoms@users.noreply.github.com> Date: Fri, 18 Nov 2022 13:13:24 +0800 Subject: [PATCH 5/5] Remove marks for sequential, vw, deeprec --- pyproject.toml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d90f5ce436..385f44302a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,14 +13,11 @@ build-backend = "setuptools.build_meta" [tool.pytest.ini_options] markers = [ - "deeprec: tests for deeprec model", - "experimental", + "experimental: tests that will not be executed and may need extra dependencies", "flaky: flaky tests that can fail unexpectedly", "gpu: tests running on GPU", "integration: integration tests", "notebooks: tests for notebooks", - "sequential", "smoke: smoke tests", "spark: tests that requires Spark", - "vw: tests for vowpal wabbit", ] \ No newline at end of file