From cc83f29d63baa4f687fc4c2487e158d66b593372 Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 18:39:23 +0000 Subject: [PATCH 01/31] add comp actiion for get test groups --- .github/actions/azureml-test/action.yml | 104 ++++++++++++++++++ .github/actions/get-test-groups/action.yml | 22 ++++ .../azureml-unit-tests-composite-actions.yml | 47 ++++++++ 3 files changed, 173 insertions(+) create mode 100644 .github/actions/azureml-test/action.yml create mode 100644 .github/actions/get-test-groups/action.yml create mode 100644 .github/workflows/azureml-unit-tests-composite-actions.yml diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml new file mode 100644 index 0000000000..ddc2e3a44a --- /dev/null +++ b/.github/actions/azureml-test/action.yml @@ -0,0 +1,104 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# --------------------------------------------------------- + +name: azureml-tests +description: "Submit experiment to AzureML cluster" +inputs: + # azureml experiment name + EXP_NAME: + required: true + type: string + # type of test - unit or nightly + TEST_KIND: + required: true + type: string + # test environment - cpu, gpu or spark + TEST_ENV: + required: false + type: string + # azureml compute credentials + AZUREML_TEST_CREDENTIALS: + required: true + type: string + # azureml compute subid + AZUREML_TEST_SUBID: + required: true + type: string + +env: + CPU_CLUSTER_NAME: "cpu-cluster" + GPU_CLUSTER_NAME: "gpu-cluster" + RG: "recommenders_project_resources" + WS: "azureml-test-workspace" + TEST_LOGS_PATH: '"test_logs.log"' + PYTEST_EXIT_CODE: "pytest_exit_code.log" + +# execute-tests: +# needs: get-test-groups +# name: ${{ join(matrix.*, ', ') }} +# runs-on: ubuntu-latest +# strategy: +# matrix: +# python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] +# test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} + +runs: + using: "composite" + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + - name: Install azureml-core and azure-cli on a GitHub hosted server + run: pip install azureml-core azure-cli + - name: Log in to Azure + uses: azure/login@v1 + with: + creds: ${{inputs.AZUREML_TEST_CREDENTIALS}} + - name: Install wheel package + run: pip install wheel + - name: Create wheel from setup.py + run: python setup.py bdist_wheel + - name: Extract branch name + shell: bash + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: extract_branch + - name: Submit CPU tests to AzureML + if: contains(matrix.test-group, 'cpu') + run: >- + python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CPU_CLUSTER_NAME}} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{inputs.EXP_NAME}}_${{matrix.test-group}} + --testlogs ${{env.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} + --conda_pkg_python ${{matrix.python-version}} --testgroup ${{matrix.test-group}} + - name: Submit GPU tests to AzureML + if: contains(matrix.test-group, 'gpu') + run: >- + python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.GPU_CLUSTER_NAME}} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{inputs.EXP_NAME}}_${{matrix.test-group}} + --testlogs ${{env.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} + --conda_pkg_python ${{matrix.python-version}} --testgroup ${{matrix.test-group}} + - name: Submit PySpark tests to AzureML + if: contains(matrix.test-group, 'spark') + run: >- + python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CPU_CLUSTER_NAME}} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{inputs.EXP_NAME}}_${{matrix.test-group}} + --testlogs ${{env.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} + --conda_pkg_python ${{matrix.python-version}} --testgroup ${{matrix.test-group}} + - name: Print test logs + run: cat ${{env.TEST_LOGS_PATH}} + - name: Get exit status + id: exit_status + run: echo ::set-output name=code::$(cat ${{env.PYTEST_EXIT_CODE}}) + - name: Check Success/Failure + if: ${{ steps.exit_status.outputs.code != 0 }} + uses: actions/github-script@v3 + with: + script: | + core.setFailed('All tests did not pass!') diff --git a/.github/actions/get-test-groups/action.yml b/.github/actions/get-test-groups/action.yml new file mode 100644 index 0000000000..3582cfa307 --- /dev/null +++ b/.github/actions/get-test-groups/action.yml @@ -0,0 +1,22 @@ +name: get-test-groups +description: "Get test group names from tests_groups.py" +inputs: + # type of test - unit or nightly + TEST_KIND: + required: true + type: string + +runs: + using: "composite" + steps: + - name: Get test group names + id: get_test_groups + run: | + if [[ ${{ inputs.TEST_KIND }} == "nightly" ]]; then + test_groups_str=$(python -c 'from tests.ci.azureml_tests.test_groups import nightly_test_groups; print([t for t in nightly_test_groups.keys() if "${{inputs.TEST_ENV}}" in t])') + else + test_groups_str=$(python -c 'from tests.ci.azureml_tests.test_groups import unit_test_groups; print(list(unit_test_groups.keys()))') + fi + echo ::set-output name=test_groups::$test_groups_str + outputs: + test_groups: ${{steps.get_test_groups.outputs.test_groups}} \ No newline at end of file diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml new file mode 100644 index 0000000000..6dc5b5a894 --- /dev/null +++ b/.github/workflows/azureml-unit-tests-composite-actions.yml @@ -0,0 +1,47 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# --------------------------------------------------------- + +name: azureml-unit-tests + +on: + # # pull_request_target allows execution of workflows in the context + # # of a base repository. When a PR is raised from a forked repo, it + # # gives the workflow access to AZUREML_TEST_CREDENTIALS, stored as + # # a repo level secret, which is needed to trigger execution of unit + # # tests on AzureML compute. + # pull_request_target: + # types: + # - opened + # - reopened + # - synchronize + # branches: + # - 'staging' + # - 'main' + + # # enable manual trigger + # workflow_dispatch: + # input: + # tags: + # description: 'Tags to label this manual run (optional)' + # default: 'Anything to describe this manual run' + + # # make this workflow reusable + # workflow_call: + + push: + branches: + - 'pradjoshi/composite_actions' + +jobs: + + get-test-groups: + outputs: + test_groups: ${{ steps.get-test-groups.output.test_groups }} + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Get test group names + id: deploy + uses: ./.github/actions/get-test-groups From 1dc394613b9469e926745fcd4c8cef50a6ea561d Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 18:41:46 +0000 Subject: [PATCH 02/31] add run on prop --- .github/workflows/azureml-unit-tests-composite-actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml index 6dc5b5a894..7282a1531c 100644 --- a/.github/workflows/azureml-unit-tests-composite-actions.yml +++ b/.github/workflows/azureml-unit-tests-composite-actions.yml @@ -35,7 +35,7 @@ on: - 'pradjoshi/composite_actions' jobs: - + runs-on: ubuntu-latest get-test-groups: outputs: test_groups: ${{ steps.get-test-groups.output.test_groups }} From e634f520b7230dbd509d04cf443e9b02783d42b7 Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 18:47:53 +0000 Subject: [PATCH 03/31] resolve syntax error --- .../workflows/azureml-unit-tests-composite-actions.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml index 7282a1531c..600e3f1c2e 100644 --- a/.github/workflows/azureml-unit-tests-composite-actions.yml +++ b/.github/workflows/azureml-unit-tests-composite-actions.yml @@ -40,8 +40,8 @@ jobs: outputs: test_groups: ${{ steps.get-test-groups.output.test_groups }} steps: - - name: Check out repository code - uses: actions/checkout@v2 - - name: Get test group names - id: deploy - uses: ./.github/actions/get-test-groups + - name: Check out repository code + uses: actions/checkout@v2 + - name: Get test group names + id: get-test-groups + uses: ./.github/actions/get-test-groups From 2e5f519833f82f74ad44f89118a88b3619e02545 Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 18:55:59 +0000 Subject: [PATCH 04/31] resolve syntax error --- .github/workflows/azureml-unit-tests-composite-actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml index 600e3f1c2e..d723775472 100644 --- a/.github/workflows/azureml-unit-tests-composite-actions.yml +++ b/.github/workflows/azureml-unit-tests-composite-actions.yml @@ -35,8 +35,8 @@ on: - 'pradjoshi/composite_actions' jobs: - runs-on: ubuntu-latest get-test-groups: + runs-on: ubuntu-latest outputs: test_groups: ${{ steps.get-test-groups.output.test_groups }} steps: From 7e2df57815cd51beb689a4323000eab63f0b9765 Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 18:59:34 +0000 Subject: [PATCH 05/31] add test kind arg --- .github/workflows/azureml-unit-tests-composite-actions.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml index d723775472..8bdaa2ca2f 100644 --- a/.github/workflows/azureml-unit-tests-composite-actions.yml +++ b/.github/workflows/azureml-unit-tests-composite-actions.yml @@ -45,3 +45,5 @@ jobs: - name: Get test group names id: get-test-groups uses: ./.github/actions/get-test-groups + with: + TEST_KIND: "unit" From c9ab2ab1c3b62e41b2a3aabea102e3eefe7abc6c Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 19:14:30 +0000 Subject: [PATCH 06/31] add bash shell --- .github/actions/get-test-groups/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/get-test-groups/action.yml b/.github/actions/get-test-groups/action.yml index 3582cfa307..d8cda463fe 100644 --- a/.github/actions/get-test-groups/action.yml +++ b/.github/actions/get-test-groups/action.yml @@ -11,6 +11,7 @@ runs: steps: - name: Get test group names id: get_test_groups + shell: bash run: | if [[ ${{ inputs.TEST_KIND }} == "nightly" ]]; then test_groups_str=$(python -c 'from tests.ci.azureml_tests.test_groups import nightly_test_groups; print([t for t in nightly_test_groups.keys() if "${{inputs.TEST_ENV}}" in t])') From c399dd58a1d1a4d4e2db409383d2811b692dac59 Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 19:36:15 +0000 Subject: [PATCH 07/31] resolve output err --- .github/actions/get-test-groups/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/get-test-groups/action.yml b/.github/actions/get-test-groups/action.yml index d8cda463fe..b65620646a 100644 --- a/.github/actions/get-test-groups/action.yml +++ b/.github/actions/get-test-groups/action.yml @@ -20,4 +20,4 @@ runs: fi echo ::set-output name=test_groups::$test_groups_str outputs: - test_groups: ${{steps.get_test_groups.outputs.test_groups}} \ No newline at end of file + test_groups: ${{steps.get_test_groups.outputs.test_groups.value}} \ No newline at end of file From 3b55c184bc3c19202a4b66189236a5f8fc4d09c8 Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 19:40:54 +0000 Subject: [PATCH 08/31] resolve output err --- .github/actions/get-test-groups/action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/get-test-groups/action.yml b/.github/actions/get-test-groups/action.yml index b65620646a..da39f593fe 100644 --- a/.github/actions/get-test-groups/action.yml +++ b/.github/actions/get-test-groups/action.yml @@ -19,5 +19,6 @@ runs: test_groups_str=$(python -c 'from tests.ci.azureml_tests.test_groups import unit_test_groups; print(list(unit_test_groups.keys()))') fi echo ::set-output name=test_groups::$test_groups_str - outputs: - test_groups: ${{steps.get_test_groups.outputs.test_groups.value}} \ No newline at end of file + +outputs: + test_groups: ${{steps.get_test_groups.outputs.test_groups}} \ No newline at end of file From 877cd0ee40ec5c34f96291d195a42a77922e860f Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 19:45:17 +0000 Subject: [PATCH 09/31] resolve output err --- .github/actions/get-test-groups/action.yml | 5 ++--- .github/workflows/azureml-unit-tests-composite-actions.yml | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/actions/get-test-groups/action.yml b/.github/actions/get-test-groups/action.yml index da39f593fe..ba5f6ba6a1 100644 --- a/.github/actions/get-test-groups/action.yml +++ b/.github/actions/get-test-groups/action.yml @@ -5,6 +5,8 @@ inputs: TEST_KIND: required: true type: string +outputs: + test_groups: ${{steps.get_test_groups.outputs.test_groups}} runs: using: "composite" @@ -19,6 +21,3 @@ runs: test_groups_str=$(python -c 'from tests.ci.azureml_tests.test_groups import unit_test_groups; print(list(unit_test_groups.keys()))') fi echo ::set-output name=test_groups::$test_groups_str - -outputs: - test_groups: ${{steps.get_test_groups.outputs.test_groups}} \ No newline at end of file diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml index 8bdaa2ca2f..b9d06a2924 100644 --- a/.github/workflows/azureml-unit-tests-composite-actions.yml +++ b/.github/workflows/azureml-unit-tests-composite-actions.yml @@ -37,8 +37,6 @@ on: jobs: get-test-groups: runs-on: ubuntu-latest - outputs: - test_groups: ${{ steps.get-test-groups.output.test_groups }} steps: - name: Check out repository code uses: actions/checkout@v2 @@ -47,3 +45,5 @@ jobs: uses: ./.github/actions/get-test-groups with: TEST_KIND: "unit" + - run: echo ${{ steps.get-test-groups.output.test_groups }} + shell: bash From a2beee38d3fdfd48d33aa058ad4afddcd31e0c3f Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 19:48:34 +0000 Subject: [PATCH 10/31] resolve output err --- .github/actions/get-test-groups/action.yml | 3 ++- .github/workflows/azureml-unit-tests-composite-actions.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/get-test-groups/action.yml b/.github/actions/get-test-groups/action.yml index ba5f6ba6a1..c4d2ba456e 100644 --- a/.github/actions/get-test-groups/action.yml +++ b/.github/actions/get-test-groups/action.yml @@ -6,7 +6,8 @@ inputs: required: true type: string outputs: - test_groups: ${{steps.get_test_groups.outputs.test_groups}} + test_groups: + value: ${{steps.get_test_groups.outputs.test_groups}} runs: using: "composite" diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml index b9d06a2924..83eeb550de 100644 --- a/.github/workflows/azureml-unit-tests-composite-actions.yml +++ b/.github/workflows/azureml-unit-tests-composite-actions.yml @@ -45,5 +45,5 @@ jobs: uses: ./.github/actions/get-test-groups with: TEST_KIND: "unit" - - run: echo ${{ steps.get-test-groups.output.test_groups }} + - run: echo ${{ steps.get-test-groups.outputs.test_groups }} shell: bash From 54ed77b027b2a617618bb9fcfd14239fc5f016c1 Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 21:39:45 +0000 Subject: [PATCH 11/31] add unit test action --- .github/actions/azureml-test/action.yml | 35 +++++++++---------- .../azureml-unit-tests-composite-actions.yml | 22 +++++++++++- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index ddc2e3a44a..067de624ab 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -26,6 +26,14 @@ inputs: AZUREML_TEST_SUBID: required: true type: string + # python version + PYTHON_VERSION: + required: true + type: string + # test group name + TEST_GROUP: + required: true + type: string env: CPU_CLUSTER_NAME: "cpu-cluster" @@ -35,15 +43,6 @@ env: TEST_LOGS_PATH: '"test_logs.log"' PYTEST_EXIT_CODE: "pytest_exit_code.log" -# execute-tests: -# needs: get-test-groups -# name: ${{ join(matrix.*, ', ') }} -# runs-on: ubuntu-latest -# strategy: -# matrix: -# python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] -# test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} - runs: using: "composite" steps: @@ -68,29 +67,29 @@ runs: run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" id: extract_branch - name: Submit CPU tests to AzureML - if: contains(matrix.test-group, 'cpu') + if: contains(inputs.TEST_GROUP, 'cpu') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CPU_CLUSTER_NAME}} --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} - --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{inputs.EXP_NAME}}_${{matrix.test-group}} + --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{env.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} - --conda_pkg_python ${{matrix.python-version}} --testgroup ${{matrix.test-group}} + --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - name: Submit GPU tests to AzureML - if: contains(matrix.test-group, 'gpu') + if: contains(inputs.TEST_GROUP, 'gpu') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.GPU_CLUSTER_NAME}} --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} - --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{inputs.EXP_NAME}}_${{matrix.test-group}} + --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{env.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} - --conda_pkg_python ${{matrix.python-version}} --testgroup ${{matrix.test-group}} + --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - name: Submit PySpark tests to AzureML - if: contains(matrix.test-group, 'spark') + if: contains(inputs.TEST_GROUP, 'spark') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CPU_CLUSTER_NAME}} --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} - --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{inputs.EXP_NAME}}_${{matrix.test-group}} + --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{env.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} - --conda_pkg_python ${{matrix.python-version}} --testgroup ${{matrix.test-group}} + --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}}} - name: Print test logs run: cat ${{env.TEST_LOGS_PATH}} - name: Get exit status diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml index 83eeb550de..ea1aadaea7 100644 --- a/.github/workflows/azureml-unit-tests-composite-actions.yml +++ b/.github/workflows/azureml-unit-tests-composite-actions.yml @@ -45,5 +45,25 @@ jobs: uses: ./.github/actions/get-test-groups with: TEST_KIND: "unit" - - run: echo ${{ steps.get-test-groups.outputs.test_groups }} + - name: Print test group names + run: echo ${{ steps.get-test-groups.outputs.test_groups }} shell: bash + + execute-tests: + needs: get-test-groups + name: ${{ join(matrix.*, ', ') }} + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] + test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} + steps: + - name: Execute tests + uses: ./.github/actions/azureml-test + with: + EXP_NAME: 'unit_tests' + TEST_KIND: 'unit' + AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }} + AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }} + PYTHON_VERSION: ${{ matrix.python-version }} + TEST_GROUP: ${{ matrix.test-group }} From 825c574b1cc83954626959dd01edcdf3a50634c7 Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 21:56:26 +0000 Subject: [PATCH 12/31] resolve json error --- .github/workflows/azureml-unit-tests-composite-actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml index ea1aadaea7..0b25f7d7e7 100644 --- a/.github/workflows/azureml-unit-tests-composite-actions.yml +++ b/.github/workflows/azureml-unit-tests-composite-actions.yml @@ -56,7 +56,7 @@ jobs: strategy: matrix: python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] - test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} + test-group: ${{ needs.get-test-groups.outputs.test_groups }} steps: - name: Execute tests uses: ./.github/actions/azureml-test From fe1031af23de1ff0f252dae41388cf7599c403dc Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 21:58:46 +0000 Subject: [PATCH 13/31] resolve json error --- .github/workflows/azureml-unit-tests-composite-actions.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml index 0b25f7d7e7..79ef28f0b3 100644 --- a/.github/workflows/azureml-unit-tests-composite-actions.yml +++ b/.github/workflows/azureml-unit-tests-composite-actions.yml @@ -48,6 +48,8 @@ jobs: - name: Print test group names run: echo ${{ steps.get-test-groups.outputs.test_groups }} shell: bash + outputs: + test_groups: ${{ steps.get_test_groups.outputs.test_groups }} execute-tests: needs: get-test-groups @@ -56,7 +58,7 @@ jobs: strategy: matrix: python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] - test-group: ${{ needs.get-test-groups.outputs.test_groups }} + test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} steps: - name: Execute tests uses: ./.github/actions/azureml-test From 7c622d0701d8e1dbf8bd41b9ddd80f85534d1b1d Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 22:06:19 +0000 Subject: [PATCH 14/31] resolve json error --- .github/workflows/azureml-unit-tests-composite-actions.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml index 79ef28f0b3..2d3de2ccdf 100644 --- a/.github/workflows/azureml-unit-tests-composite-actions.yml +++ b/.github/workflows/azureml-unit-tests-composite-actions.yml @@ -41,12 +41,12 @@ jobs: - name: Check out repository code uses: actions/checkout@v2 - name: Get test group names - id: get-test-groups + id: get_test_groups uses: ./.github/actions/get-test-groups with: TEST_KIND: "unit" - name: Print test group names - run: echo ${{ steps.get-test-groups.outputs.test_groups }} + run: echo ${{ steps.get_test_groups.outputs.test_groups }} shell: bash outputs: test_groups: ${{ steps.get_test_groups.outputs.test_groups }} From 5774ed9f1417aa1f67cdd7b5b9e5a68fe9055a9b Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 22:08:44 +0000 Subject: [PATCH 15/31] add repo checkout --- .github/actions/azureml-test/action.yml | 2 -- .github/workflows/azureml-unit-tests-composite-actions.yml | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 067de624ab..2b5d2ee22a 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -46,8 +46,6 @@ env: runs: using: "composite" steps: - - name: Check out repository code - uses: actions/checkout@v2 - name: Setup python uses: actions/setup-python@v2 with: diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml index 2d3de2ccdf..9adf7a187a 100644 --- a/.github/workflows/azureml-unit-tests-composite-actions.yml +++ b/.github/workflows/azureml-unit-tests-composite-actions.yml @@ -60,6 +60,8 @@ jobs: python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} steps: + - name: Check out repository code + uses: actions/checkout@v2 - name: Execute tests uses: ./.github/actions/azureml-test with: From 6d2934e18b7ef42b54379a8837cba3406728cc59 Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 22:12:29 +0000 Subject: [PATCH 16/31] add shell option --- .github/actions/azureml-test/action.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 2b5d2ee22a..1dfeffd426 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -51,20 +51,24 @@ runs: with: python-version: "3.8" - name: Install azureml-core and azure-cli on a GitHub hosted server + shell: bash run: pip install azureml-core azure-cli - name: Log in to Azure uses: azure/login@v1 with: creds: ${{inputs.AZUREML_TEST_CREDENTIALS}} - name: Install wheel package + shell: bash run: pip install wheel - name: Create wheel from setup.py + shell: bash run: python setup.py bdist_wheel - name: Extract branch name shell: bash run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" id: extract_branch - name: Submit CPU tests to AzureML + shell: bash if: contains(inputs.TEST_GROUP, 'cpu') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CPU_CLUSTER_NAME}} @@ -73,6 +77,7 @@ runs: --testlogs ${{env.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - name: Submit GPU tests to AzureML + shell: bash if: contains(inputs.TEST_GROUP, 'gpu') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.GPU_CLUSTER_NAME}} @@ -81,6 +86,7 @@ runs: --testlogs ${{env.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - name: Submit PySpark tests to AzureML + shell: bash if: contains(inputs.TEST_GROUP, 'spark') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CPU_CLUSTER_NAME}} @@ -89,11 +95,14 @@ runs: --testlogs ${{env.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}}} - name: Print test logs + shell: bash run: cat ${{env.TEST_LOGS_PATH}} - name: Get exit status + shell: bash id: exit_status run: echo ::set-output name=code::$(cat ${{env.PYTEST_EXIT_CODE}}) - name: Check Success/Failure + shell: bash if: ${{ steps.exit_status.outputs.code != 0 }} uses: actions/github-script@v3 with: From e8e19c3cff85796d88e9d851623f014c7fd1218c Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 22:19:36 +0000 Subject: [PATCH 17/31] fix issue with comp action --- .github/actions/azureml-test/action.yml | 8 +------- .../workflows/azureml-unit-tests-composite-actions.yml | 8 ++++++++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 1dfeffd426..9efdcadded 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -101,10 +101,4 @@ runs: shell: bash id: exit_status run: echo ::set-output name=code::$(cat ${{env.PYTEST_EXIT_CODE}}) - - name: Check Success/Failure - shell: bash - if: ${{ steps.exit_status.outputs.code != 0 }} - uses: actions/github-script@v3 - with: - script: | - core.setFailed('All tests did not pass!') + diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml index 9adf7a187a..e0845d4ef3 100644 --- a/.github/workflows/azureml-unit-tests-composite-actions.yml +++ b/.github/workflows/azureml-unit-tests-composite-actions.yml @@ -64,6 +64,7 @@ jobs: uses: actions/checkout@v2 - name: Execute tests uses: ./.github/actions/azureml-test + id: execute_tests with: EXP_NAME: 'unit_tests' TEST_KIND: 'unit' @@ -71,3 +72,10 @@ jobs: AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }} PYTHON_VERSION: ${{ matrix.python-version }} TEST_GROUP: ${{ matrix.test-group }} + - name: Check Success/Failure + shell: bash + if: ${{ steps.execute_tests.outputs.code != 0 }} + uses: actions/github-script@v3 + with: + script: | + core.setFailed('All tests did not pass!') \ No newline at end of file From b5967c065454b4c0f707bdd22f8ad410d8d57e11 Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 22:32:00 +0000 Subject: [PATCH 18/31] fix issue with comp action --- .github/actions/azureml-test/action.yml | 7 ++++++- .github/workflows/azureml-unit-tests-composite-actions.yml | 7 ------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 9efdcadded..0032043ba5 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -101,4 +101,9 @@ runs: shell: bash id: exit_status run: echo ::set-output name=code::$(cat ${{env.PYTEST_EXIT_CODE}}) - + - name: Check Success/Failure + if: ${{ steps.execute_tests.outputs.code != 0 }} + uses: actions/github-script@v3 + with: + script: | + core.setFailed('All tests did not pass!') diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml index e0845d4ef3..6536979e2e 100644 --- a/.github/workflows/azureml-unit-tests-composite-actions.yml +++ b/.github/workflows/azureml-unit-tests-composite-actions.yml @@ -72,10 +72,3 @@ jobs: AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }} PYTHON_VERSION: ${{ matrix.python-version }} TEST_GROUP: ${{ matrix.test-group }} - - name: Check Success/Failure - shell: bash - if: ${{ steps.execute_tests.outputs.code != 0 }} - uses: actions/github-script@v3 - with: - script: | - core.setFailed('All tests did not pass!') \ No newline at end of file From d02acf5a65e34ec59387c36142c4eb5615a00aa8 Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 22:42:02 +0000 Subject: [PATCH 19/31] refactor env variables into args for comp actions --- .github/actions/azureml-test/action.yml | 38 +++++++++++++++++++------ 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 0032043ba5..7629399117 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -34,14 +34,36 @@ inputs: TEST_GROUP: required: true type: string - -env: - CPU_CLUSTER_NAME: "cpu-cluster" - GPU_CLUSTER_NAME: "gpu-cluster" - RG: "recommenders_project_resources" - WS: "azureml-test-workspace" - TEST_LOGS_PATH: '"test_logs.log"' - PYTEST_EXIT_CODE: "pytest_exit_code.log" + # cpu cluster name + CPU_CLUSTER_NAME: + required: false + type: string + default: "cpu-cluster" + # gpu cluster name + GPU_CLUSTER_NAME: + required: false + type: string + default: "gpu-cluster" + # AzureML resource group name + RG: + required: false + type: string + default: "recommenders_project_resources" + # AzureML workspace name + WS: + required: false + type: string + default: "azureml-test-workspace" + # test logs path + TEST_LOGS_PATH: + required: false + type: string + default: '"test_logs.log"' + # pytest exit code + PYTEST_EXIT_CODE: + required: false + type: string + default: "pytest_exit_code.log" runs: using: "composite" From 48caf8b07a239bdf30a369ef5f43526d2bd22d01 Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 22:50:47 +0000 Subject: [PATCH 20/31] refactor env variables into args for comp actions --- .github/actions/azureml-test/action.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 7629399117..d70e6d2fdd 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -93,36 +93,36 @@ runs: shell: bash if: contains(inputs.TEST_GROUP, 'cpu') run: >- - python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CPU_CLUSTER_NAME}} + python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} - --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} - --testlogs ${{env.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} + --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} + --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - name: Submit GPU tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'gpu') run: >- - python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.GPU_CLUSTER_NAME}} + python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.GPU_CLUSTER_NAME}} --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} - --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} - --testlogs ${{env.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} + --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} + --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - name: Submit PySpark tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'spark') run: >- - python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CPU_CLUSTER_NAME}} + python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} - --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} - --testlogs ${{env.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} + --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} + --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}}} - name: Print test logs shell: bash - run: cat ${{env.TEST_LOGS_PATH}} + run: cat ${{inputs.TEST_LOGS_PATH}} - name: Get exit status shell: bash id: exit_status - run: echo ::set-output name=code::$(cat ${{env.PYTEST_EXIT_CODE}}) + run: echo ::set-output name=code::$(cat ${{inputs.PYTEST_EXIT_CODE}}) - name: Check Success/Failure if: ${{ steps.execute_tests.outputs.code != 0 }} uses: actions/github-script@v3 From e63e700ca62566f3ef00b26c5dc7c506b1f40e5f Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 23:06:08 +0000 Subject: [PATCH 21/31] remove test group typo --- .github/actions/azureml-test/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index d70e6d2fdd..d44d668200 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -115,7 +115,7 @@ runs: --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} - --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}}} + --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - name: Print test logs shell: bash run: cat ${{inputs.TEST_LOGS_PATH}} From ddddc4b03f463beb13098139c1c2eb11806cb1d2 Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 23:57:45 +0000 Subject: [PATCH 22/31] trigger nighhtly with composite actions --- .github/actions/get-test-groups/action.yml | 5 ++ .../azureml-cpu-nightly-composite-actions.yml | 73 +++++++++++++++++++ .../azureml-gpu-nightly-composite-actions.yml | 73 +++++++++++++++++++ ...zureml-spark-nightly-composite-actions.yml | 73 +++++++++++++++++++ 4 files changed, 224 insertions(+) create mode 100644 .github/workflows/azureml-cpu-nightly-composite-actions.yml create mode 100644 .github/workflows/azureml-gpu-nightly-composite-actions.yml create mode 100644 .github/workflows/azureml-spark-nightly-composite-actions.yml diff --git a/.github/actions/get-test-groups/action.yml b/.github/actions/get-test-groups/action.yml index c4d2ba456e..c741b98dad 100644 --- a/.github/actions/get-test-groups/action.yml +++ b/.github/actions/get-test-groups/action.yml @@ -5,6 +5,11 @@ inputs: TEST_KIND: required: true type: string + # test environment - cpu, gpu or spark + TEST_ENV: + required: false + type: string + default: 'cpu' outputs: test_groups: value: ${{steps.get_test_groups.outputs.test_groups}} diff --git a/.github/workflows/azureml-cpu-nightly-composite-actions.yml b/.github/workflows/azureml-cpu-nightly-composite-actions.yml new file mode 100644 index 0000000000..49201a8f05 --- /dev/null +++ b/.github/workflows/azureml-cpu-nightly-composite-actions.yml @@ -0,0 +1,73 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# --------------------------------------------------------- + +name: azureml-cpu-nightly + +on: + # ┌───────────── minute (0 - 59) + # │ ┌───────────── hour (0 - 23) + # │ │ ┌───────────── day of the month (1 - 31) + # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) + # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) + # │ │ │ │ │ + # │ │ │ │ │ + schedule: + - cron: '0 0 * * *' # basically running everyday at 12AM + # cron works with default branch (main) only: # https://github.community/t/on-schedule-per-branch/17525/2 + + push: + # because we can't schedule runs for non-main branches, + # to ensure we are running the build on the staging branch, we can add push policy for it + branches: [staging, pradjoshi/composite_actions] + + # enable manual trigger + workflow_dispatch: + input: + tags: + description: 'Tags to label this manual run (optional)' + default: 'Anything to describe this manual run' + + # make this workflow reusable + workflow_call: + +jobs: + get-test-groups: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Get test group names + id: get_test_groups + uses: ./.github/actions/get-test-groups + with: + TEST_KIND: "nightly" + - name: Print test group names + run: echo ${{ steps.get_test_groups.outputs.test_groups }} + shell: bash + outputs: + test_groups: ${{ steps.get_test_groups.outputs.test_groups }} + + execute-tests: + needs: get-test-groups + name: ${{ join(matrix.*, ', ') }} + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] + test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Execute tests + uses: ./.github/actions/azureml-test + id: execute_tests + with: + EXP_NAME: 'nightly_tests' + TEST_KIND: 'nightly' + TEST_ENV: 'cpu' + AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }} + AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }} + PYTHON_VERSION: ${{ matrix.python-version }} + TEST_GROUP: ${{ matrix.test-group }} diff --git a/.github/workflows/azureml-gpu-nightly-composite-actions.yml b/.github/workflows/azureml-gpu-nightly-composite-actions.yml new file mode 100644 index 0000000000..ce3a52db2a --- /dev/null +++ b/.github/workflows/azureml-gpu-nightly-composite-actions.yml @@ -0,0 +1,73 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# --------------------------------------------------------- + +name: azureml-gpu-nightly + +on: + # ┌───────────── minute (0 - 59) + # │ ┌───────────── hour (0 - 23) + # │ │ ┌───────────── day of the month (1 - 31) + # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) + # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) + # │ │ │ │ │ + # │ │ │ │ │ + schedule: + - cron: '0 0 * * *' # basically running everyday at 12AM + # cron works with default branch (main) only: # https://github.community/t/on-schedule-per-branch/17525/2 + + push: + # because we can't schedule runs for non-main branches, + # to ensure we are running the build on the staging branch, we can add push policy for it + branches: [staging, pradjoshi/composite_actions] + + # enable manual trigger + workflow_dispatch: + input: + tags: + description: 'Tags to label this manual run (optional)' + default: 'Anything to describe this manual run' + + # make this workflow reusable + workflow_call: + +jobs: + get-test-groups: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Get test group names + id: get_test_groups + uses: ./.github/actions/get-test-groups + with: + TEST_KIND: "nightly" + - name: Print test group names + run: echo ${{ steps.get_test_groups.outputs.test_groups }} + shell: bash + outputs: + test_groups: ${{ steps.get_test_groups.outputs.test_groups }} + + execute-tests: + needs: get-test-groups + name: ${{ join(matrix.*, ', ') }} + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] + test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Execute tests + uses: ./.github/actions/azureml-test + id: execute_tests + with: + EXP_NAME: 'nightly_tests' + TEST_KIND: 'nightly' + TEST_ENV: 'gpu' + AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }} + AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }} + PYTHON_VERSION: ${{ matrix.python-version }} + TEST_GROUP: ${{ matrix.test-group }} diff --git a/.github/workflows/azureml-spark-nightly-composite-actions.yml b/.github/workflows/azureml-spark-nightly-composite-actions.yml new file mode 100644 index 0000000000..b73bd3e2c4 --- /dev/null +++ b/.github/workflows/azureml-spark-nightly-composite-actions.yml @@ -0,0 +1,73 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# --------------------------------------------------------- + +name: azureml-spark-nightly + +on: + # ┌───────────── minute (0 - 59) + # │ ┌───────────── hour (0 - 23) + # │ │ ┌───────────── day of the month (1 - 31) + # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) + # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) + # │ │ │ │ │ + # │ │ │ │ │ + schedule: + - cron: '0 0 * * *' # basically running everyday at 12AM + # cron works with default branch (main) only: # https://github.community/t/on-schedule-per-branch/17525/2 + + push: + # because we can't schedule runs for non-main branches, + # to ensure we are running the build on the staging branch, we can add push policy for it + branches: [staging, pradjoshi/composite_actions] + + # enable manual trigger + workflow_dispatch: + input: + tags: + description: 'Tags to label this manual run (optional)' + default: 'Anything to describe this manual run' + + # make this workflow reusable + workflow_call: + +jobs: + get-test-groups: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Get test group names + id: get_test_groups + uses: ./.github/actions/get-test-groups + with: + TEST_KIND: "nightly" + - name: Print test group names + run: echo ${{ steps.get_test_groups.outputs.test_groups }} + shell: bash + outputs: + test_groups: ${{ steps.get_test_groups.outputs.test_groups }} + + execute-tests: + needs: get-test-groups + name: ${{ join(matrix.*, ', ') }} + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] + test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Execute tests + uses: ./.github/actions/azureml-test + id: execute_tests + with: + EXP_NAME: 'nightly_tests' + TEST_KIND: 'nightly' + TEST_ENV: 'spark' + AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }} + AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }} + PYTHON_VERSION: ${{ matrix.python-version }} + TEST_GROUP: ${{ matrix.test-group }} From 175bf4246b6495df39ff60bf39d44e3f18e21028 Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Thu, 14 Jul 2022 23:58:10 +0000 Subject: [PATCH 23/31] trigger nighhtly with composite actions --- .../azureml-unit-tests-composite-actions.yml | 46 +++++++++---------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml index 6536979e2e..1fe546ecb6 100644 --- a/.github/workflows/azureml-unit-tests-composite-actions.yml +++ b/.github/workflows/azureml-unit-tests-composite-actions.yml @@ -6,33 +6,29 @@ name: azureml-unit-tests on: - # # pull_request_target allows execution of workflows in the context - # # of a base repository. When a PR is raised from a forked repo, it - # # gives the workflow access to AZUREML_TEST_CREDENTIALS, stored as - # # a repo level secret, which is needed to trigger execution of unit - # # tests on AzureML compute. - # pull_request_target: - # types: - # - opened - # - reopened - # - synchronize - # branches: - # - 'staging' - # - 'main' - - # # enable manual trigger - # workflow_dispatch: - # input: - # tags: - # description: 'Tags to label this manual run (optional)' - # default: 'Anything to describe this manual run' + # pull_request_target allows execution of workflows in the context + # of a base repository. When a PR is raised from a forked repo, it + # gives the workflow access to AZUREML_TEST_CREDENTIALS, stored as + # a repo level secret, which is needed to trigger execution of unit + # tests on AzureML compute. + pull_request_target: + types: + - opened + - reopened + - synchronize + branches: + - 'staging' + - 'main' - # # make this workflow reusable - # workflow_call: + # enable manual trigger + workflow_dispatch: + input: + tags: + description: 'Tags to label this manual run (optional)' + default: 'Anything to describe this manual run' - push: - branches: - - 'pradjoshi/composite_actions' + # make this workflow reusable + workflow_call: jobs: get-test-groups: From 0a33106cf24cdc1041f3b7b86113f4890c8b8c6c Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Fri, 15 Jul 2022 00:06:08 +0000 Subject: [PATCH 24/31] add test env arg --- .github/workflows/azureml-cpu-nightly-composite-actions.yml | 1 + .github/workflows/azureml-gpu-nightly-composite-actions.yml | 1 + .github/workflows/azureml-spark-nightly-composite-actions.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/azureml-cpu-nightly-composite-actions.yml b/.github/workflows/azureml-cpu-nightly-composite-actions.yml index 49201a8f05..9d84fdedcd 100644 --- a/.github/workflows/azureml-cpu-nightly-composite-actions.yml +++ b/.github/workflows/azureml-cpu-nightly-composite-actions.yml @@ -43,6 +43,7 @@ jobs: uses: ./.github/actions/get-test-groups with: TEST_KIND: "nightly" + TEST_ENV: "cpu" - name: Print test group names run: echo ${{ steps.get_test_groups.outputs.test_groups }} shell: bash diff --git a/.github/workflows/azureml-gpu-nightly-composite-actions.yml b/.github/workflows/azureml-gpu-nightly-composite-actions.yml index ce3a52db2a..8355ffacaf 100644 --- a/.github/workflows/azureml-gpu-nightly-composite-actions.yml +++ b/.github/workflows/azureml-gpu-nightly-composite-actions.yml @@ -43,6 +43,7 @@ jobs: uses: ./.github/actions/get-test-groups with: TEST_KIND: "nightly" + TEST_ENV: "gpu" - name: Print test group names run: echo ${{ steps.get_test_groups.outputs.test_groups }} shell: bash diff --git a/.github/workflows/azureml-spark-nightly-composite-actions.yml b/.github/workflows/azureml-spark-nightly-composite-actions.yml index b73bd3e2c4..690ff8a3dc 100644 --- a/.github/workflows/azureml-spark-nightly-composite-actions.yml +++ b/.github/workflows/azureml-spark-nightly-composite-actions.yml @@ -43,6 +43,7 @@ jobs: uses: ./.github/actions/get-test-groups with: TEST_KIND: "nightly" + TEST_ENV: "spark" - name: Print test group names run: echo ${{ steps.get_test_groups.outputs.test_groups }} shell: bash From c2d8d4696fadd45fee61d5447c0797a4cb797e83 Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Fri, 15 Jul 2022 00:17:55 +0000 Subject: [PATCH 25/31] cleanup test workflow files --- .../azureml-cpu-nightly-composite-actions.yml | 74 ----------- .github/workflows/azureml-cpu-nightly.yml | 45 +++++-- .../azureml-gpu-nightly-composite-actions.yml | 74 ----------- .github/workflows/azureml-gpu-nightly.yml | 45 +++++-- ...zureml-spark-nightly-composite-actions.yml | 74 ----------- .github/workflows/azureml-spark-nightly.yml | 45 +++++-- .github/workflows/azureml-template.yml | 117 ------------------ .../azureml-unit-tests-composite-actions.yml | 70 ----------- .github/workflows/azureml-unit-tests.yml | 42 ++++++- 9 files changed, 150 insertions(+), 436 deletions(-) delete mode 100644 .github/workflows/azureml-cpu-nightly-composite-actions.yml delete mode 100644 .github/workflows/azureml-gpu-nightly-composite-actions.yml delete mode 100644 .github/workflows/azureml-spark-nightly-composite-actions.yml delete mode 100644 .github/workflows/azureml-template.yml delete mode 100644 .github/workflows/azureml-unit-tests-composite-actions.yml diff --git a/.github/workflows/azureml-cpu-nightly-composite-actions.yml b/.github/workflows/azureml-cpu-nightly-composite-actions.yml deleted file mode 100644 index 9d84fdedcd..0000000000 --- a/.github/workflows/azureml-cpu-nightly-composite-actions.yml +++ /dev/null @@ -1,74 +0,0 @@ -# --------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -# --------------------------------------------------------- - -name: azureml-cpu-nightly - -on: - # ┌───────────── minute (0 - 59) - # │ ┌───────────── hour (0 - 23) - # │ │ ┌───────────── day of the month (1 - 31) - # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) - # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) - # │ │ │ │ │ - # │ │ │ │ │ - schedule: - - cron: '0 0 * * *' # basically running everyday at 12AM - # cron works with default branch (main) only: # https://github.community/t/on-schedule-per-branch/17525/2 - - push: - # because we can't schedule runs for non-main branches, - # to ensure we are running the build on the staging branch, we can add push policy for it - branches: [staging, pradjoshi/composite_actions] - - # enable manual trigger - workflow_dispatch: - input: - tags: - description: 'Tags to label this manual run (optional)' - default: 'Anything to describe this manual run' - - # make this workflow reusable - workflow_call: - -jobs: - get-test-groups: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - name: Get test group names - id: get_test_groups - uses: ./.github/actions/get-test-groups - with: - TEST_KIND: "nightly" - TEST_ENV: "cpu" - - name: Print test group names - run: echo ${{ steps.get_test_groups.outputs.test_groups }} - shell: bash - outputs: - test_groups: ${{ steps.get_test_groups.outputs.test_groups }} - - execute-tests: - needs: get-test-groups - name: ${{ join(matrix.*, ', ') }} - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] - test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - name: Execute tests - uses: ./.github/actions/azureml-test - id: execute_tests - with: - EXP_NAME: 'nightly_tests' - TEST_KIND: 'nightly' - TEST_ENV: 'cpu' - AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }} - AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }} - PYTHON_VERSION: ${{ matrix.python-version }} - TEST_GROUP: ${{ matrix.test-group }} diff --git a/.github/workflows/azureml-cpu-nightly.yml b/.github/workflows/azureml-cpu-nightly.yml index 666d4ecbed..93db64ce6e 100644 --- a/.github/workflows/azureml-cpu-nightly.yml +++ b/.github/workflows/azureml-cpu-nightly.yml @@ -33,11 +33,42 @@ on: workflow_call: jobs: + get-test-groups: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Get test group names + id: get_test_groups + uses: ./.github/actions/get-test-groups + with: + TEST_KIND: "nightly" + TEST_ENV: "cpu" + - name: Print test group names + run: echo ${{ steps.get_test_groups.outputs.test_groups }} + shell: bash + outputs: + test_groups: ${{ steps.get_test_groups.outputs.test_groups }} - cpu-nightly-tests: - uses: ./.github/workflows/azureml-template.yml - with: - EXP_NAME: 'nightly_tests' - TEST_KIND: 'nightly' - TEST_ENV: 'cpu' - secrets: inherit + execute-tests: + needs: get-test-groups + name: ${{ join(matrix.*, ', ') }} + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] + test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Execute tests + uses: ./.github/actions/azureml-test + id: execute_tests + with: + EXP_NAME: 'nightly_tests' + TEST_KIND: 'nightly' + TEST_ENV: 'cpu' + AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }} + AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }} + PYTHON_VERSION: ${{ matrix.python-version }} + TEST_GROUP: ${{ matrix.test-group }} diff --git a/.github/workflows/azureml-gpu-nightly-composite-actions.yml b/.github/workflows/azureml-gpu-nightly-composite-actions.yml deleted file mode 100644 index 8355ffacaf..0000000000 --- a/.github/workflows/azureml-gpu-nightly-composite-actions.yml +++ /dev/null @@ -1,74 +0,0 @@ -# --------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -# --------------------------------------------------------- - -name: azureml-gpu-nightly - -on: - # ┌───────────── minute (0 - 59) - # │ ┌───────────── hour (0 - 23) - # │ │ ┌───────────── day of the month (1 - 31) - # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) - # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) - # │ │ │ │ │ - # │ │ │ │ │ - schedule: - - cron: '0 0 * * *' # basically running everyday at 12AM - # cron works with default branch (main) only: # https://github.community/t/on-schedule-per-branch/17525/2 - - push: - # because we can't schedule runs for non-main branches, - # to ensure we are running the build on the staging branch, we can add push policy for it - branches: [staging, pradjoshi/composite_actions] - - # enable manual trigger - workflow_dispatch: - input: - tags: - description: 'Tags to label this manual run (optional)' - default: 'Anything to describe this manual run' - - # make this workflow reusable - workflow_call: - -jobs: - get-test-groups: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - name: Get test group names - id: get_test_groups - uses: ./.github/actions/get-test-groups - with: - TEST_KIND: "nightly" - TEST_ENV: "gpu" - - name: Print test group names - run: echo ${{ steps.get_test_groups.outputs.test_groups }} - shell: bash - outputs: - test_groups: ${{ steps.get_test_groups.outputs.test_groups }} - - execute-tests: - needs: get-test-groups - name: ${{ join(matrix.*, ', ') }} - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] - test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - name: Execute tests - uses: ./.github/actions/azureml-test - id: execute_tests - with: - EXP_NAME: 'nightly_tests' - TEST_KIND: 'nightly' - TEST_ENV: 'gpu' - AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }} - AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }} - PYTHON_VERSION: ${{ matrix.python-version }} - TEST_GROUP: ${{ matrix.test-group }} diff --git a/.github/workflows/azureml-gpu-nightly.yml b/.github/workflows/azureml-gpu-nightly.yml index 02b247a552..3b7cd06548 100644 --- a/.github/workflows/azureml-gpu-nightly.yml +++ b/.github/workflows/azureml-gpu-nightly.yml @@ -33,11 +33,42 @@ on: workflow_call: jobs: + get-test-groups: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Get test group names + id: get_test_groups + uses: ./.github/actions/get-test-groups + with: + TEST_KIND: "nightly" + TEST_ENV: "gpu" + - name: Print test group names + run: echo ${{ steps.get_test_groups.outputs.test_groups }} + shell: bash + outputs: + test_groups: ${{ steps.get_test_groups.outputs.test_groups }} - gpu-nightly-tests: - uses: ./.github/workflows/azureml-template.yml - with: - EXP_NAME: 'nightly_tests' - TEST_KIND: 'nightly' - TEST_ENV: 'gpu' - secrets: inherit + execute-tests: + needs: get-test-groups + name: ${{ join(matrix.*, ', ') }} + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] + test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Execute tests + uses: ./.github/actions/azureml-test + id: execute_tests + with: + EXP_NAME: 'nightly_tests' + TEST_KIND: 'nightly' + TEST_ENV: 'gpu' + AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }} + AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }} + PYTHON_VERSION: ${{ matrix.python-version }} + TEST_GROUP: ${{ matrix.test-group }} diff --git a/.github/workflows/azureml-spark-nightly-composite-actions.yml b/.github/workflows/azureml-spark-nightly-composite-actions.yml deleted file mode 100644 index 690ff8a3dc..0000000000 --- a/.github/workflows/azureml-spark-nightly-composite-actions.yml +++ /dev/null @@ -1,74 +0,0 @@ -# --------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -# --------------------------------------------------------- - -name: azureml-spark-nightly - -on: - # ┌───────────── minute (0 - 59) - # │ ┌───────────── hour (0 - 23) - # │ │ ┌───────────── day of the month (1 - 31) - # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) - # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) - # │ │ │ │ │ - # │ │ │ │ │ - schedule: - - cron: '0 0 * * *' # basically running everyday at 12AM - # cron works with default branch (main) only: # https://github.community/t/on-schedule-per-branch/17525/2 - - push: - # because we can't schedule runs for non-main branches, - # to ensure we are running the build on the staging branch, we can add push policy for it - branches: [staging, pradjoshi/composite_actions] - - # enable manual trigger - workflow_dispatch: - input: - tags: - description: 'Tags to label this manual run (optional)' - default: 'Anything to describe this manual run' - - # make this workflow reusable - workflow_call: - -jobs: - get-test-groups: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - name: Get test group names - id: get_test_groups - uses: ./.github/actions/get-test-groups - with: - TEST_KIND: "nightly" - TEST_ENV: "spark" - - name: Print test group names - run: echo ${{ steps.get_test_groups.outputs.test_groups }} - shell: bash - outputs: - test_groups: ${{ steps.get_test_groups.outputs.test_groups }} - - execute-tests: - needs: get-test-groups - name: ${{ join(matrix.*, ', ') }} - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] - test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - name: Execute tests - uses: ./.github/actions/azureml-test - id: execute_tests - with: - EXP_NAME: 'nightly_tests' - TEST_KIND: 'nightly' - TEST_ENV: 'spark' - AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }} - AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }} - PYTHON_VERSION: ${{ matrix.python-version }} - TEST_GROUP: ${{ matrix.test-group }} diff --git a/.github/workflows/azureml-spark-nightly.yml b/.github/workflows/azureml-spark-nightly.yml index fb680019eb..f55bbd3eed 100644 --- a/.github/workflows/azureml-spark-nightly.yml +++ b/.github/workflows/azureml-spark-nightly.yml @@ -33,11 +33,42 @@ on: workflow_call: jobs: + get-test-groups: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Get test group names + id: get_test_groups + uses: ./.github/actions/get-test-groups + with: + TEST_KIND: "nightly" + TEST_ENV: "spark" + - name: Print test group names + run: echo ${{ steps.get_test_groups.outputs.test_groups }} + shell: bash + outputs: + test_groups: ${{ steps.get_test_groups.outputs.test_groups }} - spark-nightly-tests: - uses: ./.github/workflows/azureml-template.yml - with: - EXP_NAME: 'nightly_tests' - TEST_KIND: 'nightly' - TEST_ENV: 'spark' - secrets: inherit + execute-tests: + needs: get-test-groups + name: ${{ join(matrix.*, ', ') }} + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] + test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Execute tests + uses: ./.github/actions/azureml-test + id: execute_tests + with: + EXP_NAME: 'nightly_tests' + TEST_KIND: 'nightly' + TEST_ENV: 'spark' + AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }} + AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }} + PYTHON_VERSION: ${{ matrix.python-version }} + TEST_GROUP: ${{ matrix.test-group }} diff --git a/.github/workflows/azureml-template.yml b/.github/workflows/azureml-template.yml deleted file mode 100644 index c62980fe89..0000000000 --- a/.github/workflows/azureml-template.yml +++ /dev/null @@ -1,117 +0,0 @@ -# --------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -# --------------------------------------------------------- - -name: azureml-tests - -on: - - # make this workflow reusable - workflow_call: - inputs: - # azureml experiment name - EXP_NAME: - required: true - type: string - # type of test - unit or nightly - TEST_KIND: - required: true - type: string - # test environment - cpu, gpu or spark - TEST_ENV: - required: false - type: string - -env: - CPU_CLUSTER_NAME: "cpu-cluster" - GPU_CLUSTER_NAME: "gpu-cluster" - RG: "recommenders_project_resources" - WS: "azureml-test-workspace" - TEST_LOGS_PATH: '"test_logs.log"' - PYTEST_EXIT_CODE: "pytest_exit_code.log" - -jobs: - - get-test-groups: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - name: Get test group names - id: get_test_groups - run: | - if [[ ${{ inputs.TEST_KIND }} == "nightly" ]]; then - test_groups_str=$(python -c 'from tests.ci.azureml_tests.test_groups import nightly_test_groups; print([t for t in nightly_test_groups.keys() if "${{inputs.TEST_ENV}}" in t])') - else - test_groups_str=$(python -c 'from tests.ci.azureml_tests.test_groups import unit_test_groups; print(list(unit_test_groups.keys()))') - fi - echo ::set-output name=test_groups::$test_groups_str - outputs: - test_groups: ${{steps.get_test_groups.outputs.test_groups}} - - execute-tests: - needs: get-test-groups - name: ${{ join(matrix.*, ', ') }} - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] - test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} - - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - name: Setup python - uses: actions/setup-python@v2 - with: - python-version: "3.8" - - name: Install azureml-core and azure-cli on a GitHub hosted server - run: pip install azureml-core azure-cli - - name: Log in to Azure - uses: azure/login@v1 - with: - creds: ${{secrets.AZUREML_TEST_CREDENTIALS}} - - name: Install wheel package - run: pip install wheel - - name: Create wheel from setup.py - run: python setup.py bdist_wheel - - name: Extract branch name - shell: bash - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: extract_branch - - name: Submit CPU tests to AzureML - if: contains(matrix.test-group, 'cpu') - run: >- - python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CPU_CLUSTER_NAME}} - --subid ${{secrets.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} - --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{inputs.EXP_NAME}}_${{matrix.test-group}} - --testlogs ${{env.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} - --conda_pkg_python ${{matrix.python-version}} --testgroup ${{matrix.test-group}} - - name: Submit GPU tests to AzureML - if: contains(matrix.test-group, 'gpu') - run: >- - python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.GPU_CLUSTER_NAME}} - --subid ${{secrets.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} - --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{inputs.EXP_NAME}}_${{matrix.test-group}} - --testlogs ${{env.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} - --conda_pkg_python ${{matrix.python-version}} --testgroup ${{matrix.test-group}} - - name: Submit PySpark tests to AzureML - if: contains(matrix.test-group, 'spark') - run: >- - python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CPU_CLUSTER_NAME}} - --subid ${{secrets.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} - --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{inputs.EXP_NAME}}_${{matrix.test-group}} - --testlogs ${{env.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} - --conda_pkg_python ${{matrix.python-version}} --testgroup ${{matrix.test-group}} - - name: Print test logs - run: cat ${{env.TEST_LOGS_PATH}} - - name: Get exit status - id: exit_status - run: echo ::set-output name=code::$(cat ${{env.PYTEST_EXIT_CODE}}) - - name: Check Success/Failure - if: ${{ steps.exit_status.outputs.code != 0 }} - uses: actions/github-script@v3 - with: - script: | - core.setFailed('All tests did not pass!') diff --git a/.github/workflows/azureml-unit-tests-composite-actions.yml b/.github/workflows/azureml-unit-tests-composite-actions.yml deleted file mode 100644 index 1fe546ecb6..0000000000 --- a/.github/workflows/azureml-unit-tests-composite-actions.yml +++ /dev/null @@ -1,70 +0,0 @@ -# --------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -# --------------------------------------------------------- - -name: azureml-unit-tests - -on: - # pull_request_target allows execution of workflows in the context - # of a base repository. When a PR is raised from a forked repo, it - # gives the workflow access to AZUREML_TEST_CREDENTIALS, stored as - # a repo level secret, which is needed to trigger execution of unit - # tests on AzureML compute. - pull_request_target: - types: - - opened - - reopened - - synchronize - branches: - - 'staging' - - 'main' - - # enable manual trigger - workflow_dispatch: - input: - tags: - description: 'Tags to label this manual run (optional)' - default: 'Anything to describe this manual run' - - # make this workflow reusable - workflow_call: - -jobs: - get-test-groups: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - name: Get test group names - id: get_test_groups - uses: ./.github/actions/get-test-groups - with: - TEST_KIND: "unit" - - name: Print test group names - run: echo ${{ steps.get_test_groups.outputs.test_groups }} - shell: bash - outputs: - test_groups: ${{ steps.get_test_groups.outputs.test_groups }} - - execute-tests: - needs: get-test-groups - name: ${{ join(matrix.*, ', ') }} - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] - test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - name: Execute tests - uses: ./.github/actions/azureml-test - id: execute_tests - with: - EXP_NAME: 'unit_tests' - TEST_KIND: 'unit' - AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }} - AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }} - PYTHON_VERSION: ${{ matrix.python-version }} - TEST_GROUP: ${{ matrix.test-group }} diff --git a/.github/workflows/azureml-unit-tests.yml b/.github/workflows/azureml-unit-tests.yml index 0dcb38f279..1fe546ecb6 100644 --- a/.github/workflows/azureml-unit-tests.yml +++ b/.github/workflows/azureml-unit-tests.yml @@ -31,10 +31,40 @@ on: workflow_call: jobs: + get-test-groups: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Get test group names + id: get_test_groups + uses: ./.github/actions/get-test-groups + with: + TEST_KIND: "unit" + - name: Print test group names + run: echo ${{ steps.get_test_groups.outputs.test_groups }} + shell: bash + outputs: + test_groups: ${{ steps.get_test_groups.outputs.test_groups }} - unit-tests: - uses: ./.github/workflows/azureml-template.yml - with: - EXP_NAME: 'unit_tests' - TEST_KIND: 'unit' - secrets: inherit + execute-tests: + needs: get-test-groups + name: ${{ join(matrix.*, ', ') }} + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] + test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Execute tests + uses: ./.github/actions/azureml-test + id: execute_tests + with: + EXP_NAME: 'unit_tests' + TEST_KIND: 'unit' + AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }} + AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }} + PYTHON_VERSION: ${{ matrix.python-version }} + TEST_GROUP: ${{ matrix.test-group }} From 021af72643e96f252d6937f3b707fa89db9f4656 Mon Sep 17 00:00:00 2001 From: Simon Zhao <43029286+simonzhaoms@users.noreply.github.com> Date: Mon, 18 Jul 2022 16:37:37 +0800 Subject: [PATCH 26/31] Add path filter for unit tests --- .github/workflows/azureml-unit-tests.yml | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/azureml-unit-tests.yml b/.github/workflows/azureml-unit-tests.yml index 0dcb38f279..5923867629 100644 --- a/.github/workflows/azureml-unit-tests.yml +++ b/.github/workflows/azureml-unit-tests.yml @@ -32,7 +32,33 @@ on: jobs: + check-changes: + # Check if there are changes in the unit tests related code which + # includes: + # * examples/ + # * recommenders/ + # * tests/ + # * setup.py + runs-on: ubuntu-latest + outputs: + code: ${{ steps.filter.outputs.code }} + steps: + - uses: actions/checkout@v3 + + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + code: + - examples/** + - recommenders/** + - tests/** + - setup.py + unit-tests: + needs: check-changes + # Unit tests will be run only when related code changes + if: needs.check-changes.outputs.code == 'true' uses: ./.github/workflows/azureml-template.yml with: EXP_NAME: 'unit_tests' From 4a6754fd28f9b41556bdc882bd448fb4964d3c04 Mon Sep 17 00:00:00 2001 From: Simon Zhao <43029286+simonzhaoms@users.noreply.github.com> Date: Mon, 18 Jul 2022 17:55:44 +0800 Subject: [PATCH 27/31] Filter paths at workflow level --- .github/workflows/azureml-unit-tests.yml | 33 +++++------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/.github/workflows/azureml-unit-tests.yml b/.github/workflows/azureml-unit-tests.yml index c73a795112..e3826c29c7 100644 --- a/.github/workflows/azureml-unit-tests.yml +++ b/.github/workflows/azureml-unit-tests.yml @@ -19,6 +19,13 @@ on: branches: - 'staging' - 'main' + paths: + # Unit tests will be run only when there are changes in the + # unit tests related code including: + - examples/** + - recommenders/** + - tests/** + - setup.py # enable manual trigger workflow_dispatch: @@ -31,33 +38,7 @@ on: workflow_call: jobs: - check-changes: - # Check if there are changes in the unit tests related code which - # includes: - # * examples/ - # * recommenders/ - # * tests/ - # * setup.py - runs-on: ubuntu-latest - outputs: - code: ${{ steps.filter.outputs.code }} - steps: - - uses: actions/checkout@v3 - - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - code: - - examples/** - - recommenders/** - - tests/** - - setup.py - get-test-groups: - needs: check-changes - # Unit tests will be run only when related code changes - if: needs.check-changes.outputs.code == 'true' runs-on: ubuntu-latest steps: - name: Check out repository code From 5beaa260e4b083c66ca5ee7d80b984aae266e850 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 19 Jul 2022 11:30:54 +0200 Subject: [PATCH 28/31] release notes --- NEWS.md | 8 ++++++++ README.md | 15 +++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/NEWS.md b/NEWS.md index 5ea1b0abcc..bfd68fcebf 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,13 @@ # What's New +## Update July, 2022 + +We have a new release [Recommenders 1.1.1](https://github.com/microsoft/recommenders/releases/tag/1.1.1)! + +We have introduced a new way of testing our repository using [AzureML](https://azure.microsoft.com/en-us/services/machine-learning/). With AzureML we are able to distribute our tests to different machines and run them in parallel. This allows us to test our repository on a wider range of machines and provides us with a much faster test cycle. Our total computation time went from around 9h to 35min, and we were able to reduce the costs by half. See more details [here](tests/README.md). + +We also made other improvements like faster evaluation metrics and improving SAR algorithm. + ## Update April 1, 2022 We have a new release [Recommenders 1.1.0](https://github.com/microsoft/recommenders/releases/tag/1.1.0)! diff --git a/README.md b/README.md index faee5f0606..b580d0c254 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,13 @@ [![Documentation Status](https://readthedocs.org/projects/microsoft-recommenders/badge/?version=latest)](https://microsoft-recommenders.readthedocs.io/en/latest/?badge=latest) -## What's New (April 1, 2022) - -We have a new release [Recommenders 1.1.0](https://github.com/microsoft/recommenders/releases/tag/1.1.0)! -We have introduced the SASRec and SSEPT algorithms that are based on transformers. -In addition, we now have enabled Python 3.8 and 3.9. -We have also made improvements on the SARPlus algorithm, including support for Azure Synapse and Spark 3.2. -There are also bug fixes and improvements on NCF, RBM, LightGBM, LightFM, Scikit-Surprise, the stratified splitter, dockerfile -and upgrade to Scikit-Learn 1.0.2. +## What's New (July, 2022) + +We have a new release [Recommenders 1.1.1](https://github.com/microsoft/recommenders/releases/tag/1.1.1)! + +We have introduced a new way of testing our repository using [AzureML](https://azure.microsoft.com/en-us/services/machine-learning/). With AzureML we are able to distribute our tests to different machines and run them in parallel. This allows us to test our repository on a wider range of machines and provides us with a much faster test cycle. Our total computation time went from around 9h to 35min, and we were able to reduce the costs by half. See more details [here](tests/README.md). + +We also made other improvements like faster evaluation metrics and improving SAR algorithm. Starting with release 0.6.0, Recommenders has been available on PyPI and can be installed using pip! From 7888a682cde5d5ec3997b6992b0efc24eba27e8d Mon Sep 17 00:00:00 2001 From: pradnyeshjoshi Date: Tue, 19 Jul 2022 16:41:28 +0000 Subject: [PATCH 29/31] execute nightly builds on PR to main --- .github/workflows/azureml-cpu-nightly.yml | 20 ++++++++++++++++++++ .github/workflows/azureml-gpu-nightly.yml | 20 ++++++++++++++++++++ .github/workflows/azureml-spark-nightly.yml | 20 ++++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/.github/workflows/azureml-cpu-nightly.yml b/.github/workflows/azureml-cpu-nightly.yml index 93db64ce6e..f05482bc81 100644 --- a/.github/workflows/azureml-cpu-nightly.yml +++ b/.github/workflows/azureml-cpu-nightly.yml @@ -22,6 +22,26 @@ on: # to ensure we are running the build on the staging branch, we can add push policy for it branches: [staging] + # pull_request_target allows execution of workflows in the context + # of a base repository. When a PR is raised from a forked repo, it + # gives the workflow access to AZUREML_TEST_CREDENTIALS, stored as + # a repo level secret, which is needed to trigger execution of unit + # tests on AzureML compute. + pull_request_target: + types: + - opened + - reopened + - synchronize + branches: + - 'main' + paths: + # Unit tests will be run only when there are changes in the + # unit tests related code including: + - examples/** + - recommenders/** + - tests/** + - setup.py + # enable manual trigger workflow_dispatch: input: diff --git a/.github/workflows/azureml-gpu-nightly.yml b/.github/workflows/azureml-gpu-nightly.yml index 3b7cd06548..e5d0421f4a 100644 --- a/.github/workflows/azureml-gpu-nightly.yml +++ b/.github/workflows/azureml-gpu-nightly.yml @@ -22,6 +22,26 @@ on: # to ensure we are running the build on the staging branch, we can add push policy for it branches: [staging] + # pull_request_target allows execution of workflows in the context + # of a base repository. When a PR is raised from a forked repo, it + # gives the workflow access to AZUREML_TEST_CREDENTIALS, stored as + # a repo level secret, which is needed to trigger execution of unit + # tests on AzureML compute. + pull_request_target: + types: + - opened + - reopened + - synchronize + branches: + - 'main' + paths: + # Unit tests will be run only when there are changes in the + # unit tests related code including: + - examples/** + - recommenders/** + - tests/** + - setup.py + # enable manual trigger workflow_dispatch: input: diff --git a/.github/workflows/azureml-spark-nightly.yml b/.github/workflows/azureml-spark-nightly.yml index f55bbd3eed..35ee2a00c4 100644 --- a/.github/workflows/azureml-spark-nightly.yml +++ b/.github/workflows/azureml-spark-nightly.yml @@ -22,6 +22,26 @@ on: # to ensure we are running the build on the staging branch, we can add push policy for it branches: [staging] + # pull_request_target allows execution of workflows in the context + # of a base repository. When a PR is raised from a forked repo, it + # gives the workflow access to AZUREML_TEST_CREDENTIALS, stored as + # a repo level secret, which is needed to trigger execution of unit + # tests on AzureML compute. + pull_request_target: + types: + - opened + - reopened + - synchronize + branches: + - 'main' + paths: + # Unit tests will be run only when there are changes in the + # unit tests related code including: + - examples/** + - recommenders/** + - tests/** + - setup.py + # enable manual trigger workflow_dispatch: input: From 5ec342c2cc8549273ba03774b96e26eaa00649a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Gonz=C3=A1lez-Fierro?= <3491412+miguelgfierro@users.noreply.github.com> Date: Tue, 19 Jul 2022 19:31:52 +0100 Subject: [PATCH 30/31] Delete dependabot.yml --- .github/dependabot.yml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 342ef9e259..0000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,12 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - -version: 2 -updates: - - package-ecosystem: "pip" # See documentation for possible values - directory: "/" # Location of package manifests - schedule: - interval: "daily" - open-pull-requests-limit: 1 From 8f6722609b58169bca73d9b723b9bcd77520bf61 Mon Sep 17 00:00:00 2001 From: Chuyang Ke Date: Tue, 19 Jul 2022 14:48:21 -0700 Subject: [PATCH 31/31] Optimized Spark recall_at_k runtime --- recommenders/evaluation/spark_evaluation.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/recommenders/evaluation/spark_evaluation.py b/recommenders/evaluation/spark_evaluation.py index c63b9c6006..e366c2a56c 100644 --- a/recommenders/evaluation/spark_evaluation.py +++ b/recommenders/evaluation/spark_evaluation.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. import numpy as np + try: from pyspark.mllib.evaluation import RegressionMetrics, RankingMetrics from pyspark.sql import Window, DataFrame @@ -322,9 +323,13 @@ def recall_at_k(self): Return: float: recall at k (min=0, max=1). """ - recall = self._items_for_user_all.rdd.map( - lambda x: float(len(set(x[0]).intersection(set(x[1])))) / float(len(x[1])) - ).mean() + df_hit = self._items_for_user_all.withColumn( + "hit", F.array_intersect(DEFAULT_PREDICTION_COL, "ground_truth") + ) + df_hit = df_hit.withColumn("num_hit", F.size("hit")) + df_hit = df_hit.withColumn("num_actual", F.size("ground_truth")) + df_hit = df_hit.withColumn("per_hit", df_hit["num_hit"] / df_hit["num_actual"]) + recall = df_hit.select(F.mean("per_hit")).collect()[0][0] return recall