From ac2c05789534bd0f451a059e5b4217bbf5aa6040 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Wed, 3 Jun 2020 13:17:57 -0700 Subject: [PATCH 1/2] Compress mono builds that don't run tests to single jobs --- eng/pipelines/common/global-build-job.yml | 16 +- eng/pipelines/common/platform-matrix.yml | 1 - eng/pipelines/common/xplat-setup.yml | 15 + eng/pipelines/coreclr/templates/build-job.yml | 1 - eng/pipelines/coreclr/templates/xplat-job.yml | 15 - eng/pipelines/installer/jobs/base-job.yml | 3 - eng/pipelines/libraries/base-job.yml | 1 - eng/pipelines/runtime.yml | 263 ++++++------------ 8 files changed, 119 insertions(+), 196 deletions(-) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 29071209e79dc1..b9ee201e9841e0 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -9,6 +9,7 @@ parameters: variables: {} timeoutInMinutes: '' pool: '' + condition: true jobs: - template: /eng/common/templates/job/job.yml @@ -19,10 +20,19 @@ jobs: pool: ${{ parameters.pool }} container: ${{ parameters.container }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + condition: and(succeeded(), ${{ parameters.condition }}) workspace: clean: all - variables: ${{ parameters.variables }} + variables: + - name: _osParameter + value: -os ${{ parameters.osGroup }} + + - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubGroup, '_musl')) }}: + - name: _osParameter + value: /p:RuntimeOS=linux-musl /p:OutputRid=linux-musl-${{ parameters.archType }} + + - ${{ parameters.variables }} steps: - template: /eng/pipelines/common/clone-checkout-bundle-step.yml @@ -37,7 +47,7 @@ jobs: displayName: Disk Usage before Build # Build - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci -arch ${{ parameters.archType }} ${{ parameters.buildArgs }} + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci -arch ${{ parameters.archType }} $(_osParameter) ${{ parameters.buildArgs }} displayName: Build product - ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}: @@ -51,6 +61,6 @@ jobs: inputs: PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/' PublishLocation: Container - ArtifactName: $(Agent.Os)_PublishBuildAssets + ArtifactName: Logs_Build_${{ parameters.osGroup }}_${{ parameters.osSubGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} continueOnError: true condition: always() diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 0175989e2c8a32..b32df90d199096 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -28,7 +28,6 @@ parameters: jobs: # Linux arm - - ${{ if or(containsValue(parameters.platforms, 'Linux_arm'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - template: xplat-setup.yml parameters: diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml index 54da4233adaff3..dcc9eefc4b2ad4 100644 --- a/eng/pipelines/common/xplat-setup.yml +++ b/eng/pipelines/common/xplat-setup.yml @@ -30,6 +30,21 @@ jobs: - name: System.DisableZipDownload value: true + - ${{ if eq(parameters.jobParameters.buildConfig, 'checked') }}: + - name: buildConfigUpper + value: 'Checked' + + - ${{ if eq(parameters.jobParameters.buildConfig, 'debug') }}: + - name: buildConfigUpper + value: 'Debug' + + - ${{ if eq(parameters.jobParameters.buildConfig, 'release') }}: + - name: buildConfigUpper + value: 'Release' + + - name: _BuildConfig + value: $(buildConfigUpper) + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - name: archiveExtension value: '.zip' diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index c23db63f956668..15180a72241d2e 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -22,7 +22,6 @@ jobs: - template: xplat-pipeline-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} - _BuildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup }} diff --git a/eng/pipelines/coreclr/templates/xplat-job.yml b/eng/pipelines/coreclr/templates/xplat-job.yml index 7d83a052f8c29b..7f03b78d162f71 100644 --- a/eng/pipelines/coreclr/templates/xplat-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-job.yml @@ -72,21 +72,6 @@ jobs: - name: buildConfig value: ${{ parameters.buildConfig }} - - ${{ if eq(parameters.buildConfig, 'checked') }}: - - name: buildConfigUpper - value: 'Checked' - - - ${{ if eq(parameters.buildConfig, 'debug') }}: - - name: buildConfigUpper - value: 'Debug' - - - ${{ if eq(parameters.buildConfig, 'release') }}: - - name: buildConfigUpper - value: 'Release' - - - name: _BuildConfig - value: $(buildConfigUpper) - - name: archType value: ${{ parameters.archType }} diff --git a/eng/pipelines/installer/jobs/base-job.yml b/eng/pipelines/installer/jobs/base-job.yml index 8e9c3f1824bb20..6dc8db53f506e7 100644 --- a/eng/pipelines/installer/jobs/base-job.yml +++ b/eng/pipelines/installer/jobs/base-job.yml @@ -59,9 +59,6 @@ jobs: - name: OfficialBuildArg value: '' - - - name: _BuildConfig - value: ${{ parameters.buildConfig }} - name: SkipTests value: ${{ or( diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 0aa652b45ad37e..0a7b5bb58c4da0 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -38,7 +38,6 @@ jobs: pool: ${{ parameters.pool }} variables: - _buildScriptFileName: build - - _BuildConfig: ${{ parameters.buildConfig }} - _msbuildCommonParameters: '' # rename this variable, due to collision with build-native.proj diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 6e432e652e90b0..db8d064116efa0 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -233,130 +233,148 @@ jobs: eq(dependencies.checkout.outputs['SetPathVars_coreclr.containsChange'], true), eq(variables['isFullMatrix'], true))) -# -# Build Mono debug -# Only when libraries or mono changed +# Build the whole product using Mono runtime +# Only when libraries, mono or installer are changed # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} runtimeFlavor: mono - buildConfig: debug platforms: - - Android_x64 - Android_x86 - - Android_arm - Android_arm64 - tvOS_x64 - - tvOS_arm64 - - iOS_x64 - - iOS_x86 - - iOS_arm - iOS_arm64 - - OSX_x64 - - Linux_x64 - - Linux_arm - - Linux_arm64 - - Linux_musl_x64 + - iOS_x86 - Browser_wasm - # - Linux_musl_arm64 - - Windows_NT_x64 - # - Windows_NT_x86 - # - Windows_NT_arm - # - Windows_NT_arm64 + - Linux_arm jobParameters: + testGroup: innerloop + nameSuffix: All_Mono + buildArgs: -s mono+libs+installer -c $(_BuildConfig) + /p:StripSymbols=true /p:SkipTests=true condition: >- or( eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true), eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true), + eq(dependencies.checkout.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isFullMatrix'], true)) -# -# Build Mono release -# Only when libraries or mono changed -# - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: Release runtimeFlavor: mono - buildConfig: release platforms: - Android_x64 - - Android_x86 - Android_arm - - Android_arm64 - - tvOS_x64 - tvOS_arm64 - - iOS_x64 - - iOS_x86 - iOS_arm - - iOS_arm64 - - Linux_x64 - - Linux_arm - - Linux_arm64 + - iOS_x64 - Linux_musl_x64 - - Browser_wasm - # - Linux_musl_arm64 - - Windows_NT_x64 - # - Windows_NT_x86 - # - Windows_NT_arm - # - Windows_NT_arm64 jobParameters: + testGroup: innerloop + nameSuffix: All_Mono + buildArgs: -s mono+libs+installer -c $(_BuildConfig) + /p:StripSymbols=true /p:SkipTests=true condition: >- or( eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true), eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true), + eq(dependencies.checkout.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isFullMatrix'], true)) # -# Build Mono release -# Only when libraries, mono, or the runtime tests changed -# Currently only these architectures are needed for the runtime tests. +# Build Mono and Installer on LLVMJIT mode +# - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: Release runtimeFlavor: mono - buildConfig: release platforms: - OSX_x64 jobParameters: + testGroup: innerloop + nameSuffix: All_Mono_LLVMJIT + buildArgs: -s mono+libs+installer -c $(_BuildConfig) + /p:StripSymbols=true /p:SkipTests=true + /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=false condition: >- or( eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.checkout.outputs['SetPathVars_runtimetests.containsChange'], true), eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true), + eq(dependencies.checkout.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isFullMatrix'], true)) - +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + runtimeFlavor: mono + platforms: + - Linux_x64 + jobParameters: + testGroup: innerloop + nameSuffix: All_Mono_LLVMJIT + buildArgs: -s mono+libs+installer -c $(_BuildConfig) + /p:StripSymbols=true /p:SkipTests=true + /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=false + condition: >- + or( + eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true), + eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true), + eq(dependencies.checkout.outputs['SetPathVars_installer.containsChange'], true), + eq(variables['isFullMatrix'], true)) # -# Build Mono LLVM debug -# Only when libraries or mono changed +# Build Mono and Installer on LLVMAOT mode # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: Release runtimeFlavor: mono - buildConfig: debug platforms: - - OSX_x64 - Linux_x64 - # - Linux_arm - # - Linux_arm64 - # - Linux_musl_x64 - # - Linux_musl_arm64 - # - Windows_NT_x64 - # - Windows_NT_x86 - # - Windows_NT_arm - # - Windows_NT_arm64 jobParameters: - runtimeVariant: LLVMJIT + testGroup: innerloop + nameSuffix: All_Mono_LLVMAOT + buildArgs: -s mono+libs+installer -c $(_BuildConfig) + /p:StripSymbols=true /p:SkipTests=true + /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=true condition: >- or( eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true), eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true), + eq(dependencies.checkout.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isFullMatrix'], true)) +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + runtimeFlavor: mono + platforms: + - OSX_x64 + jobParameters: + testGroup: innerloop + nameSuffix: All_Mono_LLVMAOT + buildArgs: -s mono+libs+installer -c $(_BuildConfig) + /p:StripSymbols=true /p:SkipTests=true + /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=true + condition: >- + or( + eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true), + eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true), + eq(dependencies.checkout.outputs['SetPathVars_installer.containsChange'], true), + eq(variables['isFullMatrix'], true)) + +# +# Build Mono debug +# Only when libraries or mono changed +# - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/mono/templates/build-job.yml @@ -365,16 +383,13 @@ jobs: platforms: - OSX_x64 - Linux_x64 - # - Linux_arm - # - Linux_arm64 - # - Linux_musl_x64 + - Linux_arm64 # - Linux_musl_arm64 - # - Windows_NT_x64 + - Windows_NT_x64 # - Windows_NT_x86 # - Windows_NT_arm # - Windows_NT_arm64 jobParameters: - runtimeVariant: LLVMAOT condition: >- or( eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true), @@ -382,7 +397,7 @@ jobs: eq(variables['isFullMatrix'], true)) # -# Build Mono LLVM release +# Build Mono release # Only when libraries or mono changed # - template: /eng/pipelines/common/platform-matrix.yml @@ -391,24 +406,24 @@ jobs: runtimeFlavor: mono buildConfig: release platforms: - - OSX_x64 - Linux_x64 - # - Linux_arm - # - Linux_arm64 - # - Linux_musl_x64 + - Linux_arm64 # - Linux_musl_arm64 - # - Windows_NT_x64 + - Windows_NT_x64 # - Windows_NT_x86 # - Windows_NT_arm # - Windows_NT_arm64 jobParameters: - runtimeVariant: LLVMJIT condition: >- or( eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true), eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true), eq(variables['isFullMatrix'], true)) +# +# Build Mono release +# Only when libraries, mono, or the runtime tests changed +# Currently only these architectures are needed for the runtime tests. - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/mono/templates/build-job.yml @@ -416,20 +431,11 @@ jobs: buildConfig: release platforms: - OSX_x64 - - Linux_x64 - # - Linux_arm - # - Linux_arm64 - # - Linux_musl_x64 - # - Linux_musl_arm64 - # - Windows_NT_x64 - # - Windows_NT_x86 - # - Windows_NT_arm - # - Windows_NT_arm64 jobParameters: - runtimeVariant: LLVMAOT condition: >- or( eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true), + eq(dependencies.checkout.outputs['SetPathVars_runtimetests.containsChange'], true), eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true), eq(variables['isFullMatrix'], true)) @@ -466,38 +472,6 @@ jobs: jobParameters: liveRuntimeBuildConfig: release -# -# Build libraries using Mono CoreLib only -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Release - runtimeFlavor: mono - platforms: - - Android_x64 - - Android_arm - - tvOS_arm64 - - iOS_arm - - iOS_x64 - jobParameters: - liveRuntimeBuildConfig: release - -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - runtimeFlavor: mono - platforms: - - Android_x86 - - Android_arm64 - - tvOS_x64 - - iOS_arm64 - - iOS_x86 - - Browser_wasm - jobParameters: - liveRuntimeBuildConfig: debug - # # Libraries Build that only run when libraries is changed # @@ -583,61 +557,6 @@ jobs: liveRuntimeBuildConfig: release liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} -- template: /eng/pipelines/installer/installer-matrix.yml - parameters: - buildConfig: Release - runtimeFlavor: mono - platforms: - - Android_x64 - - Android_arm - - tvOS_arm64 - - iOS_arm - - iOS_x64 - jobParameters: - liveRuntimeBuildConfig: release - liveLibrariesBuildConfig: Release - -- template: /eng/pipelines/installer/installer-matrix.yml - parameters: - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - runtimeFlavor: mono - platforms: - - Android_x86 - - Android_arm64 - - tvOS_x64 - - iOS_arm64 - - iOS_x86 - - OSX_x64 - - Linux_x64 - - Browser_wasm - jobParameters: - liveRuntimeBuildConfig: release - liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - -- template: /eng/pipelines/installer/installer-matrix.yml - parameters: - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - runtimeFlavor: mono - platforms: - - OSX_x64 - - Linux_x64 - jobParameters: - runtimeVariant: LLVMJIT - liveRuntimeBuildConfig: release - liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - -- template: /eng/pipelines/installer/installer-matrix.yml - parameters: - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - runtimeFlavor: mono - platforms: - - OSX_x64 - - Linux_x64 - jobParameters: - runtimeVariant: LLVMAOT - liveRuntimeBuildConfig: release - liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - # # Libraries Test Build # Only when CoreCLR, Mono or Libraries is changed From 250c1a869187c4630d26068e7edcd08deb5f256d Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Thu, 4 Jun 2020 10:48:46 -0700 Subject: [PATCH 2/2] PR Feedback --- eng/pipelines/common/platform-matrix.yml | 1 + eng/pipelines/mono/templates/xplat-job.yml | 11 ----------- eng/pipelines/runtime.yml | 6 ------ 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index b32df90d199096..0175989e2c8a32 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -28,6 +28,7 @@ parameters: jobs: # Linux arm + - ${{ if or(containsValue(parameters.platforms, 'Linux_arm'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - template: xplat-setup.yml parameters: diff --git a/eng/pipelines/mono/templates/xplat-job.yml b/eng/pipelines/mono/templates/xplat-job.yml index 899f36ef85ad81..28971ca101bac9 100644 --- a/eng/pipelines/mono/templates/xplat-job.yml +++ b/eng/pipelines/mono/templates/xplat-job.yml @@ -70,17 +70,6 @@ jobs: - name: buildConfig value: ${{ parameters.buildConfig }} - - ${{ if eq(parameters.buildConfig, 'debug') }}: - - name: buildConfigUpper - value: 'Debug' - - - ${{ if eq(parameters.buildConfig, 'release') }}: - - name: buildConfigUpper - value: 'Release' - - - name: _BuildConfig - value: $(buildConfigUpper) - - name: archType value: ${{ parameters.archType }} diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index db8d064116efa0..bf55fca4fc9d00 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -253,7 +253,6 @@ jobs: testGroup: innerloop nameSuffix: All_Mono buildArgs: -s mono+libs+installer -c $(_BuildConfig) - /p:StripSymbols=true /p:SkipTests=true condition: >- or( eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true), @@ -277,7 +276,6 @@ jobs: testGroup: innerloop nameSuffix: All_Mono buildArgs: -s mono+libs+installer -c $(_BuildConfig) - /p:StripSymbols=true /p:SkipTests=true condition: >- or( eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true), @@ -299,7 +297,6 @@ jobs: testGroup: innerloop nameSuffix: All_Mono_LLVMJIT buildArgs: -s mono+libs+installer -c $(_BuildConfig) - /p:StripSymbols=true /p:SkipTests=true /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=false condition: >- or( @@ -319,7 +316,6 @@ jobs: testGroup: innerloop nameSuffix: All_Mono_LLVMJIT buildArgs: -s mono+libs+installer -c $(_BuildConfig) - /p:StripSymbols=true /p:SkipTests=true /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=false condition: >- or( @@ -342,7 +338,6 @@ jobs: testGroup: innerloop nameSuffix: All_Mono_LLVMAOT buildArgs: -s mono+libs+installer -c $(_BuildConfig) - /p:StripSymbols=true /p:SkipTests=true /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=true condition: >- or( @@ -362,7 +357,6 @@ jobs: testGroup: innerloop nameSuffix: All_Mono_LLVMAOT buildArgs: -s mono+libs+installer -c $(_BuildConfig) - /p:StripSymbols=true /p:SkipTests=true /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=true condition: >- or(