From 95b979dfc1e851bcdd138195056351a41ea8a49b Mon Sep 17 00:00:00 2001 From: Ye Jianquan Date: Thu, 27 Oct 2022 08:13:13 +0800 Subject: [PATCH] [202012][TestbedV2]Migrate t0 and t1-lag to TestbedV2 (#12383) (#12454) * [TestbedV2]Migrate t0 and t1-lag to TestbedV2 (#12383) signed-off-by: jianquanye@microsoft.com Migrate the t0 and t1-lag test jobs in buildimage repo to TestbedV2. Why I did it Migrate the t0 and t1-lag test jobs in buildimage repo to TestbedV2. How I did it Migrate the t0 and t1-lag test jobs in buildimage repo to TestbedV2. Remove ceos type setting Use 202012 branch as sonic-mgmt branch * Modify template to use 202012 branch --- .../run-test-scheduler-template.yml | 119 ++++++++++++++++++ azure-pipelines.yml | 111 ++++++++++++++-- 2 files changed, 218 insertions(+), 12 deletions(-) create mode 100644 .azure-pipelines/run-test-scheduler-template.yml diff --git a/.azure-pipelines/run-test-scheduler-template.yml b/.azure-pipelines/run-test-scheduler-template.yml new file mode 100644 index 000000000000..963b9d5b59c5 --- /dev/null +++ b/.azure-pipelines/run-test-scheduler-template.yml @@ -0,0 +1,119 @@ +parameters: +- name: TOPOLOGY + type: string + +- name: POLL_INTERVAL + type: number + default: 10 + +- name: POLL_TIMEOUT + type: number + default: 36000 + +- name: MIN_WORKER + type: number + default: 1 + +- name: MAX_WORKER + type: number + default: 2 + +- name: TEST_SET + type: string + default: "" + +- name: DEPLOY_MG_EXTRA_PARAMS + type: string + default: "" + +- name: MGMT_BRANCH + type: string + default: master + +steps: + - script: | + set -ex + # always use the test plan script from master branch. + wget -O ./.azure-pipelines/test_plan.py https://raw.githubusercontent.com/sonic-net/sonic-mgmt/master/.azure-pipelines/test_plan.py + wget -O ./.azure-pipelines/pr_test_scripts.yaml https://raw.githubusercontent.com/sonic-net/sonic-mgmt/${{ parameters.MGMT_BRANCH }}/.azure-pipelines/pr_test_scripts.yaml + displayName: Download TestbedV2 scripts + + - script: | + set -ex + pip install PyYAML + rm -f new_test_plan_id.txt + python ./.azure-pipelines/test_plan.py create -t ${{ parameters.TOPOLOGY }} -o new_test_plan_id.txt --min-worker ${{ parameters.MIN_WORKER }} --max-worker ${{ parameters.MAX_WORKER }} --test-set ${{ parameters.TEST_SET }} --kvm-build-id $(KVM_BUILD_ID) --deploy-mg-extra-params "${{ parameters.DEPLOY_MG_EXTRA_PARAMS }}" --mgmt-branch ${{ parameters.MGMT_BRANCH }} + TEST_PLAN_ID=`cat new_test_plan_id.txt` + + echo "Created test plan $TEST_PLAN_ID" + echo "Check https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID for test plan status" + echo "##vso[task.setvariable variable=TEST_PLAN_ID]$TEST_PLAN_ID" + env: + TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL) + TENANT_ID: $(TESTBED_TOOLS_MSAL_TENANT_ID) + CLIENT_ID: $(TESTBED_TOOLS_MSAL_CLIENT_ID) + CLIENT_SECRET: $(TESTBED_TOOLS_MSAL_CLIENT_SECRET) + displayName: Trigger test + + - script: | + set -ex + echo "Lock testbed" + echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com" + echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID" + # When "LOCK_TESTBED" finish, it changes into "PREPARE_TESTBED" + python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 43200 --expected-states PREPARE_TESTBED EXECUTING KVMDUMP FINISHED CANCELLED FAILED + env: + TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL) + displayName: Lock testbed + timeoutInMinutes: 240 + + - script: | + set -ex + echo "Prepare testbed" + echo "Preparing the testbed(add-topo, deploy-mg) may take 15-30 minutes. Before the testbed is ready, the progress of the test plan keeps displayed as 0, please be patient(We will improve the indication in a short time)" + echo "If the progress keeps as 0 for more than 1 hour, please cancel and retry this pipeline" + echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com" + echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID" + # When "PREPARE_TESTBED" finish, it changes into "EXECUTING" + python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 2400 --expected-states EXECUTING KVMDUMP FINISHED CANCELLED FAILED + env: + TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL) + displayName: Prepare testbed + timeoutInMinutes: 40 + + - script: | + set -ex + echo "Run test" + echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com" + echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID" + # When "EXECUTING" finish, it changes into "KVMDUMP", "FAILED", "CANCELLED" or "FINISHED" + python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 18000 --expected-states KVMDUMP FINISHED CANCELLED FAILED + env: + TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL) + displayName: Run test + timeoutInMinutes: 300 + + - script: | + set -ex + echo "KVM dump" + echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com" + echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID" + # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED" + python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 43200 --expected-states FINISHED CANCELLED FAILED + condition: succeededOrFailed() + env: + TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL) + displayName: KVM dump + timeoutInMinutes: 20 + + - script: | + set -ex + echo "Try to cancel test plan $TEST_PLAN_ID, cancelling finished test plan has no effect." + python ./.azure-pipelines/test_plan.py cancel -i "$(TEST_PLAN_ID)" + condition: always() + env: + TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL) + TENANT_ID: $(TESTBED_TOOLS_MSAL_TENANT_ID) + CLIENT_ID: $(TESTBED_TOOLS_MSAL_CLIENT_ID) + CLIENT_SECRET: $(TESTBED_TOOLS_MSAL_CLIENT_SECRET) + displayName: Finalize running test plan diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 190cdcd444c5..ed140d61a58d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -78,6 +78,7 @@ stages: dependsOn: BuildVS condition: and(succeeded(), and(ne(stageDependencies.BuildVS.outputs['vs.SetVar.SKIP_VSTEST'], 'YES'), in(dependencies.BuildVS.result, 'Succeeded', 'SucceededWithIssues'))) variables: + - group: Testbed-Tools - name: inventory value: veos_vtb - name: testbed_file @@ -135,7 +136,8 @@ stages: pool: sonictest displayName: "kvmtest-t0-part1" timeoutInMinutes: 300 - + condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_CLASSICAL_TEST, 'YES')) + continueOnError: false steps: - template: .azure-pipelines/run-test-template.yml parameters: @@ -149,7 +151,8 @@ stages: pool: sonictest displayName: "kvmtest-t0-part2" timeoutInMinutes: 300 - + condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_CLASSICAL_TEST, 'YES')) + continueOnError: false steps: - template: .azure-pipelines/run-test-template.yml parameters: @@ -159,33 +162,75 @@ stages: tbtype: t0 section: part-2 + - job: t0_testbedv2 + pool: + vmImage: 'ubuntu-20.04' + displayName: "kvmtest-t0 by TestbedV2" + timeoutInMinutes: 1080 + condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES')) + continueOnError: false + steps: + - template: .azure-pipelines/run-test-scheduler-template.yml + parameters: + TOPOLOGY: t0 + MIN_WORKER: 2 + MAX_WORKER: 3 + MGMT_BRANCH: 202012 + + - job: t0_2vlans_testbedv2 + pool: + vmImage: 'ubuntu-20.04' + displayName: "kvmtest-t0-2vlans by TestbedV2" + timeoutInMinutes: 1080 + condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES')) + continueOnError: false + steps: + - template: .azure-pipelines/run-test-scheduler-template.yml + parameters: + TOPOLOGY: t0 + TEST_SET: t0-2vlans + MAX_WORKER: 1 + DEPLOY_MG_EXTRA_PARAMS: "-e vlan_config=two_vlan_a" + MGMT_BRANCH: 202012 + - job: - pool: sonictest + pool: + vmImage: 'ubuntu-20.04' displayName: "kvmtest-t0" - timeoutInMinutes: 300 dependsOn: - t0_part1 - t0_part2 + - t0_testbedv2 + - t0_2vlans_testbedv2 condition: always() variables: resultOfPart1: $[ dependencies.t0_part1.result ] resultOfPart2: $[ dependencies.t0_part2.result ] + resultOfT0TestbedV2: $[ dependencies.t0_testbedv2.result ] + resultOfT02VlansTestbedV2: $[ dependencies.t0_2vlans_testbedv2.result ] steps: - script: | + if [ $(resultOfT0TestbedV2) == "Succeeded" ] && [ $(resultOfT02VlansTestbedV2) == "Succeeded" ]; then + echo "TestbedV2 t0 passed." + exit 0 + fi + if [ $(resultOfPart1) == "Succeeded" ] && [ $(resultOfPart2) == "Succeeded" ]; then - echo "Both job kvmtest-t0-part1 and kvmtest-t0-part2 are passed." + echo "Classic t0 jobs(both part1 and part2) passed." exit 0 - else - echo "Either job kvmtest-t0-part1 or job kvmtest-t0-part2 failed! Please check the detailed information." - exit 1 fi - - job: - pool: sonictest-t1-lag - displayName: "kvmtest-t1-lag" - timeoutInMinutes: 240 + echo "Both classic and TestbedV2 t0 jobs failed! Please check the detailed information. (Any of them passed, t0 will be considered as passed)" + exit 1 + + - job: t1_lag_classic + pool: sonictest-t1-lag + displayName: "kvmtest-t1-lag classic" + timeoutInMinutes: 400 + condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_CLASSICAL_TEST, 'YES')) + continueOnError: false steps: - template: .azure-pipelines/run-test-template.yml parameters: @@ -193,3 +238,45 @@ stages: tbname: vms-kvm-t1-lag ptf_name: ptf_vms6-2 tbtype: t1-lag + + - job: t1_lag_testbedv2 + pool: + vmImage: 'ubuntu-20.04' + displayName: "kvmtest-t1-lag by TestbedV2" + timeoutInMinutes: 600 + condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES')) + continueOnError: false + steps: + - template: .azure-pipelines/run-test-scheduler-template.yml + parameters: + TOPOLOGY: t1-lag + MIN_WORKER: 2 + MAX_WORKER: 3 + MGMT_BRANCH: 202012 + + - job: + pool: + vmImage: 'ubuntu-20.04' + displayName: "kvmtest-t1-lag" + dependsOn: + - t1_lag_classic + - t1_lag_testbedv2 + condition: always() + continueOnError: false + variables: + resultOfClassic: $[ dependencies.t1_lag_classic.result ] + resultOfTestbedV2: $[ dependencies.t1_lag_testbedv2.result ] + steps: + - script: | + if [ $(resultOfTestbedV2) == "Succeeded" ]; then + echo "TestbedV2 t1-lag passed." + exit 0 + fi + + if [ $(resultOfClassic) == "Succeeded" ]; then + echo "Classic t1-lag passed." + exit 0 + fi + + echo "Both classic and TestbedV2 t1-lag jobs failed! Please check the detailed information. (Any of them passed, t1-lag will be considered as passed)" + exit 1