AzureML tests: Durations, disable warnings, logs less verbose

.github/actions/azureml-test/action.yml

@@ -74,48 +74,47 @@ runs:
         python-version: "3.8"
     - name: Install azureml-core and azure-cli on a GitHub hosted server
       shell: bash
-      run: pip install azureml-core azure-cli
+      run: pip install --quiet "azureml-core>1,<2" "azure-cli>2,<3"
     - name: Log in to Azure
       uses: azure/login@v1
       with:
         creds: ${{inputs.AZUREML_TEST_CREDENTIALS}}
     - name: Install wheel package
       shell: bash
-      run: pip install wheel
+      run: pip install --quiet wheel
     - name: Create wheel from setup.py
       shell: bash
-      run: python setup.py bdist_wheel
-    - name: Extract branch name
-      shell: bash
-      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-      id: extract_branch
+      run: python setup.py --quiet bdist_wheel
     - name: Submit CPU tests to AzureML
       shell: bash
       if: contains(inputs.TEST_GROUP, 'cpu')
       run: >-
           python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}}
-          --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }}
+          --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref }}
           --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}}
-          --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}}
+          --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} 
           --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}}
+          --disable-warnings 
     - name: Submit GPU tests to AzureML
       shell: bash
       if: contains(inputs.TEST_GROUP, 'gpu')
       run: >-
           python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.GPU_CLUSTER_NAME}}
-          --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }}
+          --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref }}
           --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}}
           --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}}
           --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}}
+          --disable-warnings
     - name: Submit PySpark tests to AzureML
       shell: bash
       if: contains(inputs.TEST_GROUP, 'spark')
       run: >-
           python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}}
-          --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }}
+          --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref }}
           --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}}
           --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}}
           --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}}
+          --disable-warnings
     - name: Print test logs
       shell: bash
       run: cat ${{inputs.TEST_LOGS_PATH}}

.github/actions/get-test-groups/action.yml

@@ -1,3 +1,8 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# ---------------------------------------------------------
+
 name: get-test-groups
 description: "Get test group names from tests_groups.py"
 inputs:

.github/workflows/azureml-cpu-nightly.yml

@@ -66,6 +66,7 @@ jobs:
     name: ${{ join(matrix.*, ', ') }}
     runs-on: ubuntu-latest
     strategy:
+      max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration
       matrix:
         python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"']
         test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}

.github/workflows/azureml-gpu-nightly.yml

@@ -66,6 +66,7 @@ jobs:
     name: ${{ join(matrix.*, ', ') }}
     runs-on: ubuntu-latest
     strategy:
+      max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration
       matrix:
         python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"']
         test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}

.github/workflows/azureml-spark-nightly.yml

@@ -66,6 +66,7 @@ jobs:
     name: ${{ join(matrix.*, ', ') }}
     runs-on: ubuntu-latest
     strategy:
+      max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration
       matrix:
         python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"']
         test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}

.github/workflows/azureml-unit-tests.yml

@@ -53,6 +53,7 @@ jobs:
     name: ${{ join(matrix.*, ', ') }}
     runs-on: ubuntu-latest
     strategy:
+      max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration
       matrix:
         python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"']
         test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}

tests/ci/azureml_tests/run_groupwise_pytest.py

@@ -7,14 +7,12 @@
 are set otherwise.
 """
 
-import logging
-import os
 import sys
-from azureml.core import Run
+import logging
 import pytest
-import json
 import argparse
 import glob
+from azureml.core import Run
 from test_groups import nightly_test_groups, unit_test_groups
 
 if __name__ == "__main__":
@@ -37,6 +35,12 @@
         default="group_cpu_001",
         help="Group name for the tests",
     )
+    # Flag to indicate whether to turn off the warnings
+    parser.add_argument(
+        "--disable-warnings",
+        action="store_true",
+        help="Turn off warnings",
+    )
     args = parser.parse_args()
 
     if args.testkind == "nightly":
@@ -55,9 +59,14 @@
     logger.info(str(sys.version))
     logger.info("Executing tests now...")
 
-    # execute pytest command
-    pytest_exit_code = pytest.main(test_group)
-
+    # Add options to pytest command (Duration and disable warnings)
+    pytest_string = test_group + ["--durations"] + ["0"]
+    if args.disable_warnings is True:
+        pytest_string += ["--disable-warnings"]
+
+    # Execute pytest command
+    pytest_exit_code = pytest.main(pytest_string)
+
     logger.info("Test execution completed!")
 
     # log pytest exit code as a metric
@@ -70,5 +79,8 @@
     # logger.info("os.listdir files {}".format(os.listdir(".")))
 
     # upload pytest stdout file
-    logs_path = (glob.glob('**/70_driver_log.txt', recursive=True) + glob.glob('**/std_log.txt', recursive=True))[0]
-    run.upload_file(name='test_logs', path_or_stream=logs_path)
+    logs_path = (
+        glob.glob("**/70_driver_log.txt", recursive=True)
+        + glob.glob("**/std_log.txt", recursive=True)
+    )[0]
+    run.upload_file(name="test_logs", path_or_stream=logs_path)

tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py

@@ -25,6 +25,7 @@
     --reponame    (str): the Github repository name
     --branch      (str): the branch being run
                     It is also possible to put any text string in these.
+
 Example:
     Usually, this script is run by a DevOps pipeline. It can also be
     run from cmd line.
@@ -96,6 +97,7 @@ def setup_workspace(
             # create_resource_group=True,
             location=location,
             auth=cli_auth,
+            show_output=False,
         )
     return ws
 
@@ -135,20 +137,22 @@ def setup_persistent_compute_target(workspace, cluster_name, vm_size, max_nodes)
             vm_size=vm_size, max_nodes=max_nodes
         )
         cpu_cluster = ComputeTarget.create(workspace, cluster_name, compute_config)
-    cpu_cluster.wait_for_completion(show_output=True)
+    cpu_cluster.wait_for_completion(show_output=False)
     return cpu_cluster
 
 
-def create_run_config(cpu_cluster,
-                    docker_proc_type,
-                    workspace,
-                    add_gpu_dependencies,
-                    add_spark_dependencies,
-                    conda_pkg_cudatoolkit,
-                    conda_pkg_cudnn,
-                    conda_pkg_jdk,
-                    conda_pkg_python,
-                    reco_wheel_path):
+def create_run_config(
+    cpu_cluster,
+    docker_proc_type,
+    workspace,
+    add_gpu_dependencies,
+    add_spark_dependencies,
+    conda_pkg_cudatoolkit,
+    conda_pkg_cudnn,
+    conda_pkg_jdk,
+    conda_pkg_python,
+    reco_wheel_path,
+):
     """
     AzureML requires the run environment to be setup prior to submission.
     This configures a docker persistent compute.  Even though
@@ -167,6 +171,7 @@ def create_run_config(cpu_cluster,
                                         added to the conda environment, else False
             add_spark_dependencies (bool)   : True if PySpark packages should be
                                         added to the conda environment, else False
+
     Return:
           run_azuremlcompute : AzureML run config
     """
@@ -191,7 +196,9 @@ def create_run_config(cpu_cluster,
     conda_dep = CondaDependencies()
     conda_dep.add_conda_package(conda_pkg_python)
     conda_dep.add_pip_package(whl_url)
-    conda_dep.add_pip_package("pymanopt@https://github.com/pymanopt/pymanopt/archive/fb36a272cdeecb21992cfd9271eb82baafeb316d.zip")
+    conda_dep.add_pip_package(
+        "pymanopt@https://github.com/pymanopt/pymanopt/archive/fb36a272cdeecb21992cfd9271eb82baafeb316d.zip"
+    )
 
     # install extra dependencies
     if add_gpu_dependencies and add_spark_dependencies:
@@ -234,38 +241,34 @@ def create_experiment(workspace, experiment_name):
 
 
 def submit_experiment_to_azureml(
-    test, run_config, experiment, test_group, test_kind
+    test, run_config, experiment, test_group, test_kind, warnings
 ):
 
     """
     Submitting the experiment to AzureML actually runs the script.
 
     Args:
-        test         (str) - pytest script, folder/test
-                             such as ./tests/ci/run_pytest.py
-        test_folder  (str) - folder where tests to run are stored,
-                             like ./tests/unit
-        test_markers (str) - test markers used by pytest
-                             "not notebooks and not spark and not gpu"
-        run_config - environment configuration
-        experiment - instance of an Experiment, a collection of
+        test (str): Pytest script, folder/test such as ./tests/ci/run_pytest.py
+        run_config (obj): Environment configuration
+        experiment (obj): Instance of an Experiment, a collection of
                      trials where each trial is a run.
+        test_group (str): Name of the test group.
+        test_kind (str): Name of the test kind, such as nightly or unit.
+        pytestargs (str): Pytest arguments.
+
     Return:
-          run : AzureML run or trial
+          obj: AzureML run or trial
     """
 
-    project_folder = "."
+    arguments = ["--testgroup", test_group, "--testkind", test_kind]
+    if warnings is True:
+        arguments.append("--disable-warnings")
 
     script_run_config = ScriptRunConfig(
-        source_directory=project_folder,
+        source_directory=".",
         script=test,
         run_config=run_config,
-        arguments=[
-            "--testgroup",
-            test_group,
-            "--testkind",
-            test_kind,
-        ],
+        arguments=arguments,
         # docker_runtime_config=dc
     )
     run = experiment.submit(script_run_config)
@@ -370,11 +373,15 @@ def create_arg_parser():
     )
     # flag to indicate whether gpu dependencies should be included in conda env
     parser.add_argument(
-        "--add_gpu_dependencies", action="store_true", help="include packages for GPU support"
+        "--add_gpu_dependencies",
+        action="store_true",
+        help="include packages for GPU support",
     )
-    # flag to indicate whether pyspark dependencies should be included in conda env    
+    # flag to indicate whether pyspark dependencies should be included in conda env
     parser.add_argument(
-        "--add_spark_dependencies", action="store_true", help="include packages for PySpark support"
+        "--add_spark_dependencies",
+        action="store_true",
+        help="include packages for PySpark support",
     )
     # path where test logs should be downloaded
     parser.add_argument(
@@ -417,6 +424,12 @@ def create_arg_parser():
         default="unit",
         help="Test kind - nightly or unit",
     )
+    # Flag to indicate whether to turn off the warnings
+    parser.add_argument(
+        "--disable-warnings",
+        action="store_true",
+        help="Turn off warnings",
+    )
     args = parser.parse_args()
 
     return args
@@ -453,7 +466,7 @@ def create_arg_parser():
         max_nodes=args.maxnodes,
     )
 
-    wheel_list = glob.glob('./dist/*.whl')
+    wheel_list = glob.glob("./dist/*.whl")
     if not wheel_list:
         logger.error("Wheel not found!")
     logger.info("Found wheel at " + wheel_list[0])
@@ -481,6 +494,7 @@ def create_arg_parser():
         experiment=experiment,
         test_group=args.testgroup,
         test_kind=args.testkind,
+        warnings=args.disable_warnings,
     )
 
     # add helpful information to experiment on Azure
@@ -494,4 +508,4 @@ def create_arg_parser():
     # save pytest exit code
     metrics = run.get_metrics()
     with open("pytest_exit_code.log", "w") as f:
-        f.write(str(metrics.get('pytest_exit_code')))
+        f.write(str(metrics.get("pytest_exit_code")))