diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index c441fbf510..1381aae075 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -74,48 +74,47 @@ runs: python-version: "3.8" - name: Install azureml-core and azure-cli on a GitHub hosted server shell: bash - run: pip install azureml-core azure-cli + run: pip install --quiet "azureml-core>1,<2" "azure-cli>2,<3" - name: Log in to Azure uses: azure/login@v1 with: creds: ${{inputs.AZUREML_TEST_CREDENTIALS}} - name: Install wheel package shell: bash - run: pip install wheel + run: pip install --quiet wheel - name: Create wheel from setup.py shell: bash - run: python setup.py bdist_wheel - - name: Extract branch name - shell: bash - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: extract_branch + run: python setup.py --quiet bdist_wheel - name: Submit CPU tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'cpu') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} - --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} + --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} + --disable-warnings - name: Submit GPU tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'gpu') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.GPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} + --disable-warnings - name: Submit PySpark tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'spark') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} + --disable-warnings - name: Print test logs shell: bash run: cat ${{inputs.TEST_LOGS_PATH}} diff --git a/.github/actions/get-test-groups/action.yml b/.github/actions/get-test-groups/action.yml index c741b98dad..127c9576e9 100644 --- a/.github/actions/get-test-groups/action.yml +++ b/.github/actions/get-test-groups/action.yml @@ -1,3 +1,8 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# --------------------------------------------------------- + name: get-test-groups description: "Get test group names from tests_groups.py" inputs: diff --git a/.github/workflows/azureml-cpu-nightly.yml b/.github/workflows/azureml-cpu-nightly.yml index 84f3ba7cbf..dc9b19b898 100644 --- a/.github/workflows/azureml-cpu-nightly.yml +++ b/.github/workflows/azureml-cpu-nightly.yml @@ -66,6 +66,7 @@ jobs: name: ${{ join(matrix.*, ', ') }} runs-on: ubuntu-latest strategy: + max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} diff --git a/.github/workflows/azureml-gpu-nightly.yml b/.github/workflows/azureml-gpu-nightly.yml index 1af9e61be6..9684a48514 100644 --- a/.github/workflows/azureml-gpu-nightly.yml +++ b/.github/workflows/azureml-gpu-nightly.yml @@ -66,6 +66,7 @@ jobs: name: ${{ join(matrix.*, ', ') }} runs-on: ubuntu-latest strategy: + max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} diff --git a/.github/workflows/azureml-spark-nightly.yml b/.github/workflows/azureml-spark-nightly.yml index f73a939731..6a9e50f817 100644 --- a/.github/workflows/azureml-spark-nightly.yml +++ b/.github/workflows/azureml-spark-nightly.yml @@ -66,6 +66,7 @@ jobs: name: ${{ join(matrix.*, ', ') }} runs-on: ubuntu-latest strategy: + max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} diff --git a/.github/workflows/azureml-unit-tests.yml b/.github/workflows/azureml-unit-tests.yml index 03bde31f38..9d1c3e5074 100644 --- a/.github/workflows/azureml-unit-tests.yml +++ b/.github/workflows/azureml-unit-tests.yml @@ -53,6 +53,7 @@ jobs: name: ${{ join(matrix.*, ', ') }} runs-on: ubuntu-latest strategy: + max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} diff --git a/tests/ci/azureml_tests/run_groupwise_pytest.py b/tests/ci/azureml_tests/run_groupwise_pytest.py index 96472b7d52..719c9022ce 100644 --- a/tests/ci/azureml_tests/run_groupwise_pytest.py +++ b/tests/ci/azureml_tests/run_groupwise_pytest.py @@ -7,14 +7,12 @@ are set otherwise. """ -import logging -import os import sys -from azureml.core import Run +import logging import pytest -import json import argparse import glob +from azureml.core import Run from test_groups import nightly_test_groups, unit_test_groups if __name__ == "__main__": @@ -37,6 +35,12 @@ default="group_cpu_001", help="Group name for the tests", ) + # Flag to indicate whether to turn off the warnings + parser.add_argument( + "--disable-warnings", + action="store_true", + help="Turn off warnings", + ) args = parser.parse_args() if args.testkind == "nightly": @@ -55,9 +59,14 @@ logger.info(str(sys.version)) logger.info("Executing tests now...") - # execute pytest command - pytest_exit_code = pytest.main(test_group) - + # Add options to pytest command (Duration and disable warnings) + pytest_string = test_group + ["--durations"] + ["0"] + if args.disable_warnings is True: + pytest_string += ["--disable-warnings"] + + # Execute pytest command + pytest_exit_code = pytest.main(pytest_string) + logger.info("Test execution completed!") # log pytest exit code as a metric @@ -70,5 +79,8 @@ # logger.info("os.listdir files {}".format(os.listdir("."))) # upload pytest stdout file - logs_path = (glob.glob('**/70_driver_log.txt', recursive=True) + glob.glob('**/std_log.txt', recursive=True))[0] - run.upload_file(name='test_logs', path_or_stream=logs_path) + logs_path = ( + glob.glob("**/70_driver_log.txt", recursive=True) + + glob.glob("**/std_log.txt", recursive=True) + )[0] + run.upload_file(name="test_logs", path_or_stream=logs_path) diff --git a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py index 8c189ebb43..b365e672ba 100644 --- a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py +++ b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py @@ -25,6 +25,7 @@ --reponame (str): the Github repository name --branch (str): the branch being run It is also possible to put any text string in these. + Example: Usually, this script is run by a DevOps pipeline. It can also be run from cmd line. @@ -96,6 +97,7 @@ def setup_workspace( # create_resource_group=True, location=location, auth=cli_auth, + show_output=False, ) return ws @@ -135,20 +137,22 @@ def setup_persistent_compute_target(workspace, cluster_name, vm_size, max_nodes) vm_size=vm_size, max_nodes=max_nodes ) cpu_cluster = ComputeTarget.create(workspace, cluster_name, compute_config) - cpu_cluster.wait_for_completion(show_output=True) + cpu_cluster.wait_for_completion(show_output=False) return cpu_cluster -def create_run_config(cpu_cluster, - docker_proc_type, - workspace, - add_gpu_dependencies, - add_spark_dependencies, - conda_pkg_cudatoolkit, - conda_pkg_cudnn, - conda_pkg_jdk, - conda_pkg_python, - reco_wheel_path): +def create_run_config( + cpu_cluster, + docker_proc_type, + workspace, + add_gpu_dependencies, + add_spark_dependencies, + conda_pkg_cudatoolkit, + conda_pkg_cudnn, + conda_pkg_jdk, + conda_pkg_python, + reco_wheel_path, +): """ AzureML requires the run environment to be setup prior to submission. This configures a docker persistent compute. Even though @@ -167,6 +171,7 @@ def create_run_config(cpu_cluster, added to the conda environment, else False add_spark_dependencies (bool) : True if PySpark packages should be added to the conda environment, else False + Return: run_azuremlcompute : AzureML run config """ @@ -191,7 +196,9 @@ def create_run_config(cpu_cluster, conda_dep = CondaDependencies() conda_dep.add_conda_package(conda_pkg_python) conda_dep.add_pip_package(whl_url) - conda_dep.add_pip_package("pymanopt@https://github.com/pymanopt/pymanopt/archive/fb36a272cdeecb21992cfd9271eb82baafeb316d.zip") + conda_dep.add_pip_package( + "pymanopt@https://github.com/pymanopt/pymanopt/archive/fb36a272cdeecb21992cfd9271eb82baafeb316d.zip" + ) # install extra dependencies if add_gpu_dependencies and add_spark_dependencies: @@ -234,38 +241,34 @@ def create_experiment(workspace, experiment_name): def submit_experiment_to_azureml( - test, run_config, experiment, test_group, test_kind + test, run_config, experiment, test_group, test_kind, warnings ): """ Submitting the experiment to AzureML actually runs the script. Args: - test (str) - pytest script, folder/test - such as ./tests/ci/run_pytest.py - test_folder (str) - folder where tests to run are stored, - like ./tests/unit - test_markers (str) - test markers used by pytest - "not notebooks and not spark and not gpu" - run_config - environment configuration - experiment - instance of an Experiment, a collection of + test (str): Pytest script, folder/test such as ./tests/ci/run_pytest.py + run_config (obj): Environment configuration + experiment (obj): Instance of an Experiment, a collection of trials where each trial is a run. + test_group (str): Name of the test group. + test_kind (str): Name of the test kind, such as nightly or unit. + pytestargs (str): Pytest arguments. + Return: - run : AzureML run or trial + obj: AzureML run or trial """ - project_folder = "." + arguments = ["--testgroup", test_group, "--testkind", test_kind] + if warnings is True: + arguments.append("--disable-warnings") script_run_config = ScriptRunConfig( - source_directory=project_folder, + source_directory=".", script=test, run_config=run_config, - arguments=[ - "--testgroup", - test_group, - "--testkind", - test_kind, - ], + arguments=arguments, # docker_runtime_config=dc ) run = experiment.submit(script_run_config) @@ -370,11 +373,15 @@ def create_arg_parser(): ) # flag to indicate whether gpu dependencies should be included in conda env parser.add_argument( - "--add_gpu_dependencies", action="store_true", help="include packages for GPU support" + "--add_gpu_dependencies", + action="store_true", + help="include packages for GPU support", ) - # flag to indicate whether pyspark dependencies should be included in conda env + # flag to indicate whether pyspark dependencies should be included in conda env parser.add_argument( - "--add_spark_dependencies", action="store_true", help="include packages for PySpark support" + "--add_spark_dependencies", + action="store_true", + help="include packages for PySpark support", ) # path where test logs should be downloaded parser.add_argument( @@ -417,6 +424,12 @@ def create_arg_parser(): default="unit", help="Test kind - nightly or unit", ) + # Flag to indicate whether to turn off the warnings + parser.add_argument( + "--disable-warnings", + action="store_true", + help="Turn off warnings", + ) args = parser.parse_args() return args @@ -453,7 +466,7 @@ def create_arg_parser(): max_nodes=args.maxnodes, ) - wheel_list = glob.glob('./dist/*.whl') + wheel_list = glob.glob("./dist/*.whl") if not wheel_list: logger.error("Wheel not found!") logger.info("Found wheel at " + wheel_list[0]) @@ -481,6 +494,7 @@ def create_arg_parser(): experiment=experiment, test_group=args.testgroup, test_kind=args.testkind, + warnings=args.disable_warnings, ) # add helpful information to experiment on Azure @@ -494,4 +508,4 @@ def create_arg_parser(): # save pytest exit code metrics = run.get_metrics() with open("pytest_exit_code.log", "w") as f: - f.write(str(metrics.get('pytest_exit_code'))) + f.write(str(metrics.get("pytest_exit_code")))