From 0d9fa657163b7a3d297240a88a27c982ec7e62a0 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 5 Jan 2021 12:06:13 +1100 Subject: [PATCH 01/10] Initial cut of swap to 1ES pools. --- .../templates/jobs/archetype-sdk-client.yml | 114 +++++++++++------- .../stages/archetype-java-release.yml | 30 ++++- 2 files changed, 97 insertions(+), 47 deletions(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 8d11b922f108..38560a561d8c 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -1,43 +1,74 @@ parameters: - SDKType: client - ServiceDirectory: not-specified # Set a default that breaks in obvious ways. - TestPipeline: false - Artifacts: [] - AdditionalModules: [] - PreTestSteps: [] - TestOptions: '$(DefaultOptions) -DtrimStackTrace=false -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -Dparallel-test-playback' - TestGoals: verify - TestMatrix: - Linux - Java 8: - OSVmImage: 'ubuntu-18.04' - JavaTestVersion: '1.8' - RunTitle: 'Linux on Java 1.8' - macOS - Java 8: - OSVmImage: 'macOS-10.15' - JavaTestVersion: '1.8' - RunTitle: 'macOS on Java 1.8' - Windows - Java 8: - OSVmImage: 'windows-2019' - JavaTestVersion: '1.8' - RunTitle: 'Windows on Java 1.8' - Linux - Java 11: - OSVmImage: 'ubuntu-18.04' - JavaTestVersion: '1.11' - RunTitle: 'Linux on Java 1.11' - macOS - Java 11: - OSVmImage: 'macOS-10.15' - JavaTestVersion: '1.11' - RunTitle: 'macOS on Java 1.11' - Windows - Java 11: - OSVmImage: 'windows-2019' - JavaTestVersion: '1.11' - RunTitle: 'Windows on Java 1.11' - Linux From Source - Java 11: - OSVmImage: 'ubuntu-18.04' - JavaTestVersion: '1.11' - RunTitle: 'From Source: Linux on Java 1.11' - AdditionalOptions: '-amd' - TestFromSource: true + - name: SDKType + type: string + default: client + - name: ServiceDirectory + type: string + default: 'not-specified' # Set a default that breaks in obvious ways. + - name: TestPipeline + type: string + default: false + - name: Artifacts + type: object + default: [] + - name: AdditionalModules + type: object + default: [] + - name: PreTestSteps + type: object + default: [] + - name: TestOptions + type: string + default: '$(DefaultOptions) -DtrimStackTrace=false -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -Dparallel-test-playback' + - name: TestGoals + type: string + default: verify + - name: WindowsPool + type: string + default: azsdk-pool-mms-win-2019-general + - name: LinuxPool + type: string + default: azsdk-pool-mms-ubuntu-1804-general + - name: TestMatrix + type: object + default: + Linux - Java 8: + Pool: ${{ parameters.LinuxPool }} + OSVmImage: + JavaTestVersion: '1.8' + RunTitle: 'Linux on Java 1.8' + macOS - Java 8: + Pool: + OSVmImage: 'macOS-10.15' + JavaTestVersion: '1.8' + RunTitle: 'macOS on Java 1.8' + Windows - Java 8: + Pool: ${{ parameters.WindowsPool }} + OSVmImage: 'windows-2019' + JavaTestVersion: '1.8' + RunTitle: 'Windows on Java 1.8' + Linux - Java 11: + Pool: ${{ parameters.LinuxPool }} + OSVmImage: + JavaTestVersion: '1.11' + RunTitle: 'Linux on Java 1.11' + macOS - Java 11: + Pool: + OSVmImage: 'macOS-10.15' + JavaTestVersion: '1.11' + RunTitle: 'macOS on Java 1.11' + Windows - Java 11: + Pool: ${{ parameters.WindowsPool }} + OSVmImage: + JavaTestVersion: '1.11' + RunTitle: 'Windows on Java 1.11' + Linux From Source - Java 11: + Pool: ${{ parameters.LinuxPool }} + OSVmImage: + JavaTestVersion: '1.11' + RunTitle: 'From Source: Linux on Java 1.11' + AdditionalOptions: '-amd' + TestFromSource: true jobs: - job: 'Build' @@ -46,7 +77,7 @@ jobs: - template: ../variables/globals.yml pool: - vmImage: 'ubuntu-18.04' + name: ${{ parameters.LinuxPool }} strategy: matrix: @@ -148,7 +179,7 @@ jobs: value: 'ubuntu-18.04' pool: - vmImage: $(OSVmImage) + pool: ${{ parameters.LinuxPool }} steps: - pwsh: | @@ -243,6 +274,7 @@ jobs: ${{ insert }}: ${{ matrixEntry.value }} pool: + name: $(Pool) vmImage: $(OSVmImage) steps: diff --git a/eng/pipelines/templates/stages/archetype-java-release.yml b/eng/pipelines/templates/stages/archetype-java-release.yml index 776eea344475..73bdc467854b 100644 --- a/eng/pipelines/templates/stages/archetype-java-release.yml +++ b/eng/pipelines/templates/stages/archetype-java-release.yml @@ -1,10 +1,28 @@ parameters: - Artifacts: [] - TestPipeline: false - ArtifactName: 'not-specified' - TargetDocRepoOwner: '' - TargetDocRepoName: '' - EnableIntegrationStage: true + - name: Artifacts + type: object + default: [] + - name: TestPipeline + type: boolean + default: false + - name: ArtifactName + type: string + default: 'not-specified' + - name: TargetDocRepoOwner + type: string + default: '' + - name: TargetDocRepoName + type: string + default: '' + - name: EnableIntegrationStage + type: boolean + default: true + - name: WindowsPool + type: string + default: azsdk-pool-mms-win-2019-general + - name: LinuxPool + type: string + default: azsdk-pool-mms-ubuntu-1804-general stages: # The signing stage is responsible for submitting binaries to ESRP for our official signing From a8d871aac29bd413b4e9acf3291e41b2bd1eb2f7 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 5 Jan 2021 13:06:44 +1100 Subject: [PATCH 02/10] Hack to work around parameter usage. --- .../templates/jobs/archetype-sdk-client.yml | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 38560a561d8c..00ecd2bf3a03 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -33,22 +33,22 @@ parameters: type: object default: Linux - Java 8: - Pool: ${{ parameters.LinuxPool }} + Pool: $(LinuxPool) OSVmImage: JavaTestVersion: '1.8' RunTitle: 'Linux on Java 1.8' macOS - Java 8: Pool: - OSVmImage: 'macOS-10.15' + OS: 'macOS-10.15' JavaTestVersion: '1.8' RunTitle: 'macOS on Java 1.8' Windows - Java 8: - Pool: ${{ parameters.WindowsPool }} - OSVmImage: 'windows-2019' + Pool: $(WindowsPool) + OSVmImage: JavaTestVersion: '1.8' RunTitle: 'Windows on Java 1.8' Linux - Java 11: - Pool: ${{ parameters.LinuxPool }} + Pool: $(LinuxPool) OSVmImage: JavaTestVersion: '1.11' RunTitle: 'Linux on Java 1.11' @@ -58,18 +58,27 @@ parameters: JavaTestVersion: '1.11' RunTitle: 'macOS on Java 1.11' Windows - Java 11: - Pool: ${{ parameters.WindowsPool }} + Pool: $(WindowsPool) OSVmImage: JavaTestVersion: '1.11' RunTitle: 'Windows on Java 1.11' Linux From Source - Java 11: - Pool: ${{ parameters.LinuxPool }} + Pool: $(LinuxPool) OSVmImage: JavaTestVersion: '1.11' RunTitle: 'From Source: Linux on Java 1.11' AdditionalOptions: '-amd' TestFromSource: true +# HACK: We can't reference parameters to define other parameters, so I am +# setting some variables with the value of parameters which can then +# be used later in the test evaluations. +variables: + - name: WindowsPool + value: ${{parameters.WindowsPool}} + - name: LinuxPool + value: ${{parameters.LinuxPool}} + jobs: - job: 'Build' @@ -175,8 +184,6 @@ jobs: variables: - template: ../variables/globals.yml - - name: OSVmImage - value: 'ubuntu-18.04' pool: pool: ${{ parameters.LinuxPool }} From aa8ed62dd33ae7b359932a6bdfcfaae4d2f0f51e Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 5 Jan 2021 13:09:34 +1100 Subject: [PATCH 03/10] Move variables into test job. --- .../templates/jobs/archetype-sdk-client.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 00ecd2bf3a03..4bcc968b15c0 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -70,14 +70,6 @@ parameters: AdditionalOptions: '-amd' TestFromSource: true -# HACK: We can't reference parameters to define other parameters, so I am -# setting some variables with the value of parameters which can then -# be used later in the test evaluations. -variables: - - name: WindowsPool - value: ${{parameters.WindowsPool}} - - name: LinuxPool - value: ${{parameters.LinuxPool}} jobs: - job: 'Build' @@ -269,8 +261,15 @@ jobs: - job: 'Test' condition: and(succeeded(), ne(variables['Skip.Test'], 'true')) + # HACK: We can't reference parameters to define other parameters, so I am + # setting some variables with the value of parameters which can then + # be used later in the test evaluations. variables: - template: ../variables/globals.yml + - name: WindowsPool + value: ${{parameters.WindowsPool}} + - name: LinuxPool + value: ${{parameters.LinuxPool}} strategy: matrix: From 793fc5361cb076d44ebc8bbdd6144cae898adec9 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 5 Jan 2021 13:14:10 +1100 Subject: [PATCH 04/10] Try using coalesce function. --- eng/pipelines/templates/jobs/archetype-sdk-client.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 4bcc968b15c0..9425bc6e05a3 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -280,8 +280,8 @@ jobs: ${{ insert }}: ${{ matrixEntry.value }} pool: - name: $(Pool) - vmImage: $(OSVmImage) + name: $[coalesce(variables['Pool'], '')] + vmImage: $[coalesce(variables['OSVmImage'], '')] steps: - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml From abdd8e613e711e428166c8d718423c5596bbe092 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 5 Jan 2021 13:16:55 +1100 Subject: [PATCH 05/10] Fix up variables for macOS matrix. --- eng/pipelines/templates/jobs/archetype-sdk-client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 9425bc6e05a3..d5005803de02 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -39,7 +39,7 @@ parameters: RunTitle: 'Linux on Java 1.8' macOS - Java 8: Pool: - OS: 'macOS-10.15' + OSVmImage: 'macOS-10.15' JavaTestVersion: '1.8' RunTitle: 'macOS on Java 1.8' Windows - Java 8: From 24d3e5de328f70a2fb5340085f9b9d9a6dae5fdc Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 5 Jan 2021 13:26:35 +1100 Subject: [PATCH 06/10] Remove verify agent OS. --- eng/pipelines/templates/jobs/archetype-sdk-client.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index d5005803de02..2a4b0f9d54a2 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -284,8 +284,6 @@ jobs: vmImage: $[coalesce(variables['OSVmImage'], '')] steps: - - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml - - template: ../steps/cache-maven-repository.yml - template: ../steps/install-reporting-tools.yml From dab820bb21bdc28e6f0fde82d41bf0c0d745e76d Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 5 Jan 2021 15:04:16 +1100 Subject: [PATCH 07/10] Fix pool selection for analyze job. --- eng/pipelines/templates/jobs/archetype-sdk-client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 2a4b0f9d54a2..d3d2944a5638 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -178,7 +178,7 @@ jobs: - template: ../variables/globals.yml pool: - pool: ${{ parameters.LinuxPool }} + name: ${{ parameters.LinuxPool }} steps: - pwsh: | From 29aee891f8534e27212d04aca8b3ff2c031ad186 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 7 Jan 2021 18:14:35 +1100 Subject: [PATCH 08/10] Try a hack to work around image for Java builds. --- eng/pipelines/templates/variables/globals.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/templates/variables/globals.yml b/eng/pipelines/templates/variables/globals.yml index 8d9627b1d78a..2a3d5892ce0d 100644 --- a/eng/pipelines/templates/variables/globals.yml +++ b/eng/pipelines/templates/variables/globals.yml @@ -13,3 +13,4 @@ variables: JavaBuildVersion: '1.11' # This is the default Java test version. It's the version used when running tests. JavaTestVersion: '1.11' + JAVA_HOME_11_X64: /usr/lib/jvm/adoptopenjdk-11-hotspot-amd64 # HACK: To work around image issue. From 6c985ec2d63c3a671790caf072235c254c688348 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Sat, 9 Jan 2021 19:14:20 +1100 Subject: [PATCH 09/10] Remove hack. --- eng/pipelines/templates/variables/globals.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/templates/variables/globals.yml b/eng/pipelines/templates/variables/globals.yml index 2a3d5892ce0d..8d9627b1d78a 100644 --- a/eng/pipelines/templates/variables/globals.yml +++ b/eng/pipelines/templates/variables/globals.yml @@ -13,4 +13,3 @@ variables: JavaBuildVersion: '1.11' # This is the default Java test version. It's the version used when running tests. JavaTestVersion: '1.11' - JAVA_HOME_11_X64: /usr/lib/jvm/adoptopenjdk-11-hotspot-amd64 # HACK: To work around image issue. From 71d9ce84e96a40dab877aeff400f1d2e4491e24a Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 12 Jan 2021 10:08:41 +1100 Subject: [PATCH 10/10] Fix type of TestPipeline parameter. --- eng/pipelines/templates/jobs/archetype-sdk-client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index d3d2944a5638..98f009bf65ca 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -6,7 +6,7 @@ parameters: type: string default: 'not-specified' # Set a default that breaks in obvious ways. - name: TestPipeline - type: string + type: boolean default: false - name: Artifacts type: object