Build all managed coreclr tests on OSX in CI

docs/workflow/testing/coreclr/test-configuration.md

@@ -37,15 +37,27 @@ Test cases are categorized by priority level. The most important subset should b
         // The .NET Foundation licenses this file to you under the MIT license.
         // See the LICENSE file in the project root for more information.
     ```
-* Disable building of a test by conditionally setting the `<DisableProjectBuild>` property.
-    * e.g. `<DisableProjectBuild Condition=" '$(Platform)' == 'arm64' ">true</DisableProjectBuild>`
+* The managed portion of all tests should be able to build on any platform.
+In fact in CI the managed portion of all tests will be built on OSX.
+Each target will run a subset of the tests built on OSX.
+Therefore the managed portion of each test **must not contain**:
+  * Target platform dependent conditionally compiled code.
+  * Target platform dependent conditionally included files
+  * Target platform dependent conditional `<DefineConstants/>`
+* Disable building and running a test on selected Targets by conditionally setting the `<CLRTestTargetUnsupported>` property.
+    * e.g. `<CLRTestTargetUnsupported Condition="'$(TargetArchitecture)' == 'arm64'">true</CLRTestTargetUnsupported>`
+* Disable building of a test by unconditionally setting the `<DisableProjectBuild>` property.
+    * e.g. `<DisableProjectBuild>true</DisableProjectBuild>`
 * Exclude test from GCStress runs by adding the following to the csproj:
     * `<GCStressIncompatible>true</GCStressIncompatible>`
 * Exclude test from JIT stress runs runs by adding the following to the csproj:
     * `<JitOptimizationSensitive>true</JitOptimizationSensitive>`
 * Add NuGet references by updating the following [test project](https://github.com/dotnet/runtime/blob/master/src/coreclr/tests/src/Common/test_dependencies/test_dependencies.csproj).
 * Get access to System.Private.CoreLib types and methods that are not exposed via public surface by adding the following to the csproj:
     * `<ReferenceSystemPrivateCoreLib>true</ReferenceSystemPrivateCoreLib>`
+* Any System.Private.CoreLib types and methods used by tests must be available for building on all platforms.
+This means there must be enough implementation for the C# compiler to find the referenced types and methods. Unsupported target platforms
+should simply `throw new PlatformNotSupportedException()` in its dummy method implementations.
 * Update exclusion list at [tests/issues.targets](https://github.com/dotnet/runtime/blob/master/src/coreclr/tests/issues.targets) if the test fails due to active bug.
 
 ### Creating a C# test project

docs/workflow/testing/coreclr/unix-test-instructions.md

@@ -55,21 +55,26 @@ Please use the following command for help.
 
 ### Unsupported and temporarily disabled tests
 
-Unsupported tests outside of Windows have two annotations in their csproj to
-ignore them when run.
+To support building all tests for all targets on single target, we use
+the conditional property
 
 ```xml
-<TestUnsupportedOutsideWindows>true</TestUnsupportedOutsideWindows>
+<CLRTestTargetUnsupported Condition="...">true</CLRTestTargetUnsupported>
 ```
 
-This will write in the bash target to skip the test by returning a passing value if run outside Windows.
+This property disables building of a test in a default build. It also
+disables running a test in the bash/batch wrapper scripts. It allows the
+test to be built on any target in CI when the `allTargets` option is
+passed to the `build-test.*` scripts.
+
+Tests which never should be built or run are marked
 
-In addition:
 ```xml
-<DisableProjectBuild Condition="'$(TargetsUnix)' == 'true'">true</DisableProjectBuild>
+<DisableProjectBuild>true</DisableProjectBuild>
 ```
 
-Is used to disable the build, that way if building on Unix cycles are saved building/running.
+This propoerty should not be conditioned on Target properties to allow
+all tests to be built for `allTargets`.
 
 PAL tests
 ---------

eng/pipelines/common/platform-matrix.yml

@@ -19,7 +19,6 @@ parameters:
   helixQueuesTemplate: ''
   stagedBuild: false
   # When set to false, suppresses reuse of OSX managed build artifacts (for pipelines without an OSX obj)
-  managedOsxBuild: true
   # When set to true, passes the 'platforms' value as a job parameter also named 'platforms'.
   # Handled as an opt-in parameter to avoid excessive yaml.
   passPlatforms: false
@@ -47,7 +46,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: Linux
         crossrootfsDir: '/crossrootfs/arm'
         ${{ insert }}: ${{ parameters.jobParameters }}
 
@@ -71,7 +69,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: Linux
         crossrootfsDir: '/crossrootfs/arm64'
         ${{ insert }}: ${{ parameters.jobParameters }}
 
@@ -100,11 +97,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        ${{ if ne(parameters.managedOsxBuild, true) }}:
-          managedTestBuildOsGroup: Linux
-          managedTestBuildOsSubgroup: _musl
-        ${{ if eq(parameters.managedOsxBuild, true) }}:
-          managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Linux musl arm64
@@ -128,7 +120,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: Linux
         crossrootfsDir: '/crossrootfs/arm64'
         ${{ insert }}: ${{ parameters.jobParameters }}
 
@@ -152,10 +143,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        ${{ if ne(parameters.managedOsxBuild, true) }}:
-          managedTestBuildOsGroup: Linux
-        ${{ if eq(parameters.managedOsxBuild, true) }}:
-          managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # WebAssembly
@@ -305,7 +292,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # tvOS arm64
@@ -325,7 +311,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # iOS x64
@@ -345,7 +330,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # iOS arm
@@ -365,7 +349,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # iOS arm64
@@ -385,7 +368,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # macOS x64
@@ -405,7 +387,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Windows x64
@@ -425,7 +406,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: Windows_NT
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Windows x86
@@ -445,7 +425,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: Windows_NT
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Windows arm
@@ -464,7 +443,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: Windows_NT
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Windows arm64
@@ -484,5 +462,4 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: Windows_NT
         ${{ insert }}: ${{ parameters.jobParameters }}

eng/pipelines/common/templates/runtimes/build-test-job.yml

@@ -38,8 +38,6 @@ jobs:
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
     osSubgroup: ${{ parameters.osSubgroup }}
-    managedTestBuildOsGroup: ${{ parameters.osGroup }}
-    managedTestBuildOsSubgroup: ${{ parameters.osSubgroup }}
     container: ${{ parameters.container }}
     testGroup: ${{ parameters.testGroup }}
     stagedBuild: ${{ parameters.stagedBuild }}
@@ -51,13 +49,13 @@ jobs:
     ${{ if eq(variables['System.TeamProject'], 'internal') }}:
       continueOnError: true
 
+    # Compute job name from template parameters
     ${{ if eq(parameters.testGroup, 'innerloop') }}:
-      name: '${{ parameters.runtimeFlavor }}_common_test_build_p0_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'      
-      displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri0 Test Build ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
-
+      name: '${{ parameters.runtimeFlavor }}_common_test_build_p0_AnyOS_AnyCPU_${{ parameters.buildConfig }}'
+      displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri0 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}'
     ${{ if ne(parameters.testGroup, 'innerloop') }}:
-      name: '${{ parameters.runtimeFlavor }}_common_test_build_p1_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'      
-      displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri1 Test Build ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
+      name: '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{ parameters.buildConfig }}'
+      displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri1 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}'
 
     # Since the condition is being altered, merge the default with the additional conditions.
     # See https://docs.microsoft.com/azure/devops/pipelines/process/conditions
@@ -116,7 +114,7 @@ jobs:
         displayName: Disk Usage before Build
 
     # Build managed test components
-    - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipnative skipgeneratelayout skiptestwrappers $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg)
+    - script: $(coreClrRepoRootDir)build-test$(scriptExt) allTargets skipnative skipgeneratelayout skiptestwrappers $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg)
       displayName: Build managed test components
 
     - ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}: 
@@ -130,11 +128,11 @@ jobs:
       parameters:
         rootFolder: $(managedTestArtifactRootFolderPath)
         includeRootFolder: false
-        archiveType: $(archiveType)
-        tarCompression: $(tarCompression)
-        archiveExtension: $(archiveExtension)
-        artifactName: $(managedTestArtifactName)
-        displayName: 'managed test components'
+        archiveExtension: '.tar.gz'
+        archiveType: tar
+        tarCompression: gz
+        artifactName: $(managedGenericTestArtifactName)
+        displayName: 'managed test components (generic)'
 
 
     # Publish .packages/microsoft.net.sdk.il needed for traversing
@@ -143,9 +141,9 @@ jobs:
       parameters:
         rootFolder: $(microsoftNetSdkIlFolderPath)
         includeRootFolder: false
-        archiveType: $(archiveType)
-        tarCompression: $(tarCompression)
-        archiveExtension: $(archiveExtension)
+        archiveType: tar
+        tarCompression: gz
+        archiveExtension: '.tar.gz'
         artifactName: $(microsoftNetSdkIlArtifactName)
         displayName: 'Microsoft.NET.Sdk.IL package'
 
@@ -155,6 +153,6 @@ jobs:
       displayName: Publish Logs
       inputs:
         targetPath: $(Build.SourcesDirectory)/artifacts/log
-        artifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.testGroup }}'
+        artifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_AnyOS_AnyCPU_$(buildConfig)_Lib${{ parameters.liveLibrariesBuildConfig }}_${{ parameters.testGroup }}'
       continueOnError: true
       condition: always()

eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -3,8 +3,6 @@ parameters:
   archType: ''
   osGroup: ''
   osSubgroup: ''
-  managedTestBuildOsGroup: ''
-  managedTestBuildOsSubgroup: ''
   container: ''
   testGroup: ''
   crossrootfsDir: ''
@@ -38,8 +36,6 @@ jobs:
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
     osSubgroup: ${{ parameters.osSubgroup }}
-    managedTestBuildOsGroup: ${{ parameters.managedTestBuildOsGroup }}
-    managedTestBuildOsSubgroup: ${{ parameters.managedTestBuildOsSubgroup }}
     container: ${{ parameters.container }}
     testGroup: ${{ parameters.testGroup }}
     crossrootfsDir: ${{ parameters.crossrootfsDir }}
@@ -57,9 +53,9 @@ jobs:
     dependsOn:
     - ${{ if ne(parameters.corefxTests, true) }}:
       - ${{ if eq(parameters.testGroup, 'innerloop') }}:
-        - '${{ parameters.runtimeFlavor }}_common_test_build_p0_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
+        - '${{ parameters.runtimeFlavor }}_common_test_build_p0_AnyOS_AnyCPU_${{parameters.buildConfig }}'
       - ${{ if ne(parameters.testGroup, 'innerloop') }}:
-        - '${{ parameters.runtimeFlavor }}_common_test_build_p1_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
+        - '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{parameters.buildConfig }}'
     - ${{ if ne(parameters.stagedBuild, true) }}:
       - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
       - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
@@ -199,9 +195,9 @@ jobs:
       - template: /eng/pipelines/common/download-artifact-step.yml
         parameters:
           unpackFolder: '$(managedTestArtifactRootFolderPath)'
-          artifactFileName: '$(managedTestArtifactName)$(archiveExtension)'
-          artifactName: '$(managedTestArtifactName)'
-          displayName: 'managed test artifacts (built on ${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }})'
+          artifactFileName: '$(managedGenericTestArtifactName).tar.gz'
+          artifactName: '$(managedGenericTestArtifactName)'
+          displayName: 'generic managed test artifacts'
 
 
     # Download product binaries directory
@@ -222,13 +218,14 @@ jobs:
           artifactName: '$(coreClrProductArtifactName)'
           displayName: 'CoreCLR product download for Mono'
 
+
     # Download and unzip the Microsoft.NET.Sdk.IL package needed for traversing
     # ilproj test projects during copynativeonly.
     - ${{ if ne(parameters.corefxTests, true) }}:
       - template: /eng/pipelines/common/download-artifact-step.yml
         parameters:
           unpackFolder: '$(microsoftNetSdkIlFolderPath)'
-          artifactFileName: '$(microsoftNetSdkIlArtifactName)$(archiveExtension)'
+          artifactFileName: '$(microsoftNetSdkIlArtifactName).tar.gz'
           artifactName: '$(microsoftNetSdkIlArtifactName)'
           displayName: 'Microsoft.NET.Sdk.IL package'
 

eng/pipelines/coreclr/ci.yml

@@ -97,13 +97,7 @@ jobs:
     jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
     buildConfig: checked
     platforms:
-    - Linux_arm
-    - Linux_arm64
     - OSX_x64
-    - Windows_NT_arm
-    - Windows_NT_arm64
-    - Windows_NT_x64
-    - Windows_NT_x86
     testGroup: outerloop
     jobParameters:
       liveLibrariesBuildConfig: Release

eng/pipelines/coreclr/crossgen2-composite.yml

@@ -33,7 +33,6 @@ jobs:
     buildConfig: checked
     platforms:
     - OSX_x64
-    - Windows_NT_x64
     jobParameters:
       testGroup: innerloop
       liveLibrariesBuildConfig: Release

eng/pipelines/coreclr/crossgen2-outerloop.yml

@@ -33,7 +33,6 @@ jobs:
     buildConfig: checked
     platforms:
     - OSX_x64
-    - Windows_NT_x64
     jobParameters:
       testGroup: outerloop
       liveLibrariesBuildConfig: Release

eng/pipelines/coreclr/crossgen2.yml

@@ -33,7 +33,6 @@ jobs:
     buildConfig: checked
     platforms:
     - OSX_x64
-    - Windows_NT_x64
     jobParameters:
       testGroup: innerloop
       liveLibrariesBuildConfig: Release

eng/pipelines/coreclr/gc-longrunning.yml

@@ -25,6 +25,7 @@ jobs:
     - Linux_arm64
     - Windows_NT_x64
     - Windows_NT_arm64
+    - OSX_x64
     jobParameters:
       testGroup: gc-longrunning
 
@@ -33,10 +34,7 @@ jobs:
     jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
     buildConfig: release
     platforms:
-    - Linux_x64
-    - Linux_arm64
-    - Windows_NT_x64
-    - Windows_NT_arm64
+    - OSX_x64
     jobParameters:
       testGroup: gc-longrunning
       liveLibrariesBuildConfig: Release
@@ -52,7 +50,6 @@ jobs:
     - Windows_NT_arm64
     helixQueueGroup: ci
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
-    managedOsxBuild: false
     jobParameters:
       testGroup: gc-longrunning
       liveLibrariesBuildConfig: Release