Split superpmi-diffs pipeline runs (#84521)

Run tpdiffs and asmdiffs on separate Helix machines, in parallel, to improve overall job throughput.
dotnet · Apr 11, 2023 · 4ebecb0 · 4ebecb0
1 parent 476bce3
commit 4ebecb0
Show file tree

Hide file tree

Showing 9 changed files with 614 additions and 392 deletions.
diff --git a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml
@@ -29,6 +29,7 @@ parameters:
   runtimeVariant: ''
   shouldContinueOnError: false
   SuperPmiCollect: ''
+  SuperPmiDiffType: ''
 
 
 steps:
@@ -62,6 +63,7 @@ steps:
       RuntimeFlavor: ${{ parameters.runtimeFlavor }}
       _RuntimeVariant: ${{ parameters.runtimeVariant }}
       _SuperPmiCollect: ${{ parameters.SuperPmiCollect }}
+      _SuperPmiDiffType: ${{ parameters.SuperPmiDiffType }}
       ${{ if eq(parameters.publishTestResults, 'true') }}:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken)
       # TODO: remove NUGET_PACKAGES once https://github.com/dotnet/arcade/issues/1578 is fixed

diff --git a/eng/pipelines/coreclr/superpmi-diffs.yml b/eng/pipelines/coreclr/superpmi-diffs.yml
@@ -65,4 +65,18 @@ extends:
           helixQueueGroup: ci
           helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
           jobParameters:
-            condition: not(eq(dependencies.evaluate_paths.outputs['SetPathVars_jiteeversionguid.containsChange'], true))
+            condition: not(eq(dependencies.evaluate_paths.outputs['SetPathVars_jiteeversionguid.containsChange'], true))
+            diffType: asmdiffs
+
+      - template: /eng/pipelines/common/platform-matrix.yml
+        parameters:
+          jobTemplate: /eng/pipelines/coreclr/templates/superpmi-diffs-job.yml
+          buildConfig: checked
+          platforms:
+          - windows_x64
+          - windows_x86
+          helixQueueGroup: ci
+          helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+          jobParameters:
+            condition: not(eq(dependencies.evaluate_paths.outputs['SetPathVars_jiteeversionguid.containsChange'], true))
+            diffType: tpdiff
diff --git a/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml
@@ -16,6 +16,7 @@ parameters:
   liveLibrariesBuildConfig: ''    # optional -- live-live libraries configuration to use for the run
   helixQueues: ''                 # required -- Helix queues
   dependOnEvaluatePaths: false
+  diffType: 'asmdiffs'            # required -- 'asmdiffs', 'tpdiff', or 'all'
 
 jobs:
 - template: xplat-pipeline-job.yml
@@ -41,6 +42,9 @@ jobs:
     - ${{ each variable in parameters.variables }}:
       - ${{insert}}: ${{ variable }}
 
+    - name: diffType
+      value: ${{ parameters.diffType }}
+
     - name: PythonScript
       value: 'py -3'
     - name: PipScript
@@ -49,11 +53,23 @@ jobs:
       value: '$(Build.SourcesDirectory)\artifacts\spmi\'
     - name: SpmiLogsLocation
       value: '$(Build.SourcesDirectory)\artifacts\spmi_logs\'
-    - name: SpmiAsmdiffsLocation
-      value: '$(Build.SourcesDirectory)\artifacts\spmi_asmdiffs\'
+    - name: SpmiDiffsLocation
+      value: '$(Build.SourcesDirectory)\artifacts\spmi_diffs\'
     - name: HelixResultLocation
       value: '$(Build.SourcesDirectory)\artifacts\helixresults\'
 
+    - name: SetupScriptDirs
+      value: ''
+    - ${{ if eq(parameters.diffType, 'asmdiffs') }}:
+      - name: SetupScriptDirs
+        value: '-checked_directory $(buildProductRootFolderPath)'
+    - ${{ if eq(parameters.diffType, 'tpdiff') }}:
+      - name: SetupScriptDirs
+        value: '-release_directory $(releaseProductRootFolderPath)'
+    - ${{ if eq(parameters.diffType, 'all') }}:
+      - name: SetupScriptDirs
+        value: '-checked_directory $(buildProductRootFolderPath) -release_directory $(releaseProductRootFolderPath)'
+
     workspace:
       clean: all
     pool:
@@ -65,11 +81,11 @@ jobs:
     - script: |
         mkdir $(SpmiCollectionLocation)
         mkdir $(SpmiLogsLocation)
-        mkdir $(SpmiAsmdiffsLocation)
+        mkdir $(SpmiDiffsLocation)
       displayName: Create directories
 
-    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_diffs_setup.py -source_directory $(Build.SourcesDirectory) -checked_directory $(buildProductRootFolderPath) -release_directory $(releaseProductRootFolderPath) -arch $(archType)
-      displayName: ${{ format('SuperPMI diffs setup ({0})', parameters.archType) }}
+    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_diffs_setup.py -source_directory $(Build.SourcesDirectory) $(SetupScriptDirs) -type $(diffType) -arch $(archType)
+      displayName: ${{ format('SuperPMI diffs setup ({0} {1})', parameters.diffType, parameters.archType) }}
 
       # Run superpmi-diffs.py script in helix
     - template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml
@@ -87,6 +103,7 @@ jobs:
         BuildConfig: ${{ parameters.buildConfig }}
         osGroup: ${{ parameters.osGroup }}
         archType: ${{ parameters.archType }}
+        SuperPmiDiffType: ${{ parameters.diffType }}
 
       # Always upload the available logs for diagnostics
     - task: CopyFiles@2
@@ -102,41 +119,42 @@ jobs:
       inputs:
         sourceFolder: '$(HelixResultLocation)'
         contents: '**/superpmi_*.md'
-        targetFolder: '$(SpmiAsmdiffsLocation)'
+        targetFolder: '$(SpmiDiffsLocation)'
       condition: always()
 
-    - task: CopyFiles@2
-      displayName: Copying dasm files of all partitions
-      inputs:
-        sourceFolder: '$(HelixResultLocation)'
-        contents: '**/Asmdiffs_*.zip'
-        targetFolder: '$(SpmiAsmdiffsLocation)'
-      condition: always()
+    - ${{ if eq(parameters.diffType, 'asmdiffs') }}:
+      - task: CopyFiles@2
+        displayName: Copying dasm files of all partitions
+        inputs:
+          sourceFolder: '$(HelixResultLocation)'
+          contents: '**/Asmdiffs_*.zip'
+          targetFolder: '$(SpmiDiffsLocation)'
+        condition: always()
 
-    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_diffs_summarize.py -diff_summary_dir $(SpmiAsmdiffsLocation) -arch $(archType)
-      displayName: ${{ format('Summarize ({0})', parameters.archType) }}
+    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_diffs_summarize.py -diff_summary_dir $(SpmiDiffsLocation) -type $(diffType) -arch $(archType)
+      displayName: ${{ format('Summarize ({0} {1})', parameters.diffType, parameters.archType) }}
       condition: always()
 
     - task: PublishPipelineArtifact@1
       displayName: Publish SuperPMI logs
       inputs:
         targetPath: $(SpmiLogsLocation)
-        artifactName: 'SuperPMI_Logs_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)'
+        artifactName: 'SuperPMI_Logs_$(diffType)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)'
       condition: always()
       continueOnError: true
 
     - task: PublishPipelineArtifact@1
-      displayName: Publish SuperPMI asmdiffs files
+      displayName: Publish SuperPMI diffs files
       inputs:
-        targetPath: $(SpmiAsmdiffsLocation)
-        artifactName: 'SuperPMI_Asmdiffs_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)'
+        targetPath: $(SpmiDiffsLocation)
+        artifactName: 'SuperPMI_Diffs_$(diffType)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)'
       condition: always()
       continueOnError: true
 
     - task: PublishPipelineArtifact@1
       displayName: Publish SuperPMI build logs
       inputs:
         targetPath: $(Build.SourcesDirectory)/artifacts/log
-        artifactName: 'SuperPMI_BuildLogs_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)'
+        artifactName: 'SuperPMI_BuildLogs_$(diffType)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)'
       condition: always()
       continueOnError: true
diff --git a/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml b/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml
@@ -10,12 +10,13 @@ parameters:
   helixQueues: ''
   dependOnEvaluatePaths: false
   runJobTemplate: '/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml'
+  diffType: 'asmdiffs'            # required -- 'asmdiffs', 'tpdiff', or 'all'
 
 jobs:
 - template: ${{ parameters.runJobTemplate }}
   parameters:
-    jobName: ${{ format('superpmi_diffs_{0}{1}_{2}', parameters.osGroup, parameters.osSubgroup, parameters.archType) }}
-    displayName: ${{ format('SuperPMI diffs {0}{1} {2}', parameters.osGroup, parameters.osSubgroup, parameters.archType) }}
+    jobName: ${{ format('superpmi_diffs_{0}_{1}{2}_{3}', parameters.diffType, parameters.osGroup, parameters.osSubgroup, parameters.archType) }}
+    displayName: ${{ format('SuperPMI {0} {1}{2} {3}', parameters.diffType, parameters.osGroup, parameters.osSubgroup, parameters.archType) }}
     pool: ${{ parameters.pool }}
     buildConfig: ${{ parameters.buildConfig }}
     archType: ${{ parameters.archType }}
@@ -25,33 +26,39 @@ jobs:
     dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }}
     timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
     helixQueues: ${{ parameters.helixQueues }}
+    diffType: ${{ parameters.diffType }}
     dependsOn:
-     - ${{ format('coreclr_jit_build_{0}{1}_{2}_checked', parameters.osGroup, parameters.osSubgroup, parameters.archType) }}
-     - ${{ format('coreclr_jit_build_{0}{1}_{2}_release', parameters.osGroup, parameters.osSubgroup, parameters.archType) }}
+      - ${{ if in(parameters.diffType, 'asmdiffs', 'all') }}:
+        - ${{ format('coreclr_jit_build_{0}{1}_{2}_checked', parameters.osGroup, parameters.osSubgroup, parameters.archType) }}
+      - ${{ if in(parameters.diffType, 'tpdiff', 'all') }}:
+        - ${{ format('coreclr_jit_build_{0}{1}_{2}_release', parameters.osGroup, parameters.osSubgroup, parameters.archType) }}
 
     variables:
     - ${{ each variable in parameters.variables }}:
       - ${{insert}}: ${{ variable }}
 
-    - name: releaseProductRootFolderPath
-      value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).Release'
-    - name: releaseProductArtifactName
-      value: 'CoreCLRProduct_${{ parameters.pgoType }}_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_release'
+    - ${{ if in(parameters.diffType, 'tpdiff', 'all') }}:
+      - name: releaseProductRootFolderPath
+        value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).Release'
+      - name: releaseProductArtifactName
+        value: 'CoreCLRProduct_${{ parameters.pgoType }}_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_release'
 
     steps:
 
-    # Download jit checked builds
-    - template: /eng/pipelines/common/download-artifact-step.yml
-      parameters:
-        unpackFolder: $(buildProductRootFolderPath)
-        artifactFileName: '$(buildProductArtifactName)$(archiveExtension)'
-        artifactName: '$(buildProductArtifactName)'
-        displayName: 'JIT checked build'
+    - ${{ if in(parameters.diffType, 'asmdiffs', 'all') }}:
+      # Download jit checked builds
+      - template: /eng/pipelines/common/download-artifact-step.yml
+        parameters:
+          unpackFolder: $(buildProductRootFolderPath)
+          artifactFileName: '$(buildProductArtifactName)$(archiveExtension)'
+          artifactName: '$(buildProductArtifactName)'
+          displayName: 'JIT checked build'
 
-    #Download jit release builds
-    - template: /eng/pipelines/common/download-artifact-step.yml
-      parameters:
-        unpackFolder: $(releaseProductRootFolderPath)
-        artifactFileName: '$(releaseProductArtifactName)$(archiveExtension)'
-        artifactName: '$(releaseProductArtifactName)'
-        displayName: 'JIT release build'
+    - ${{ if in(parameters.diffType, 'tpdiff', 'all') }}:
+      # Download jit release builds
+      - template: /eng/pipelines/common/download-artifact-step.yml
+        parameters:
+          unpackFolder: $(releaseProductRootFolderPath)
+          artifactFileName: '$(releaseProductArtifactName)$(archiveExtension)'
+          artifactName: '$(releaseProductArtifactName)'
+          displayName: 'JIT release build'
diff --git a/src/coreclr/scripts/superpmi-diffs.proj b/src/coreclr/scripts/superpmi-diffs.proj
@@ -30,8 +30,6 @@
     <SuperpmiLogsLocation>%HELIX_WORKITEM_UPLOAD_ROOT%</SuperpmiLogsLocation>
     <!-- Workaround until https://github.com/dotnet/arcade/pull/6179 is not available -->
     <HelixResultsDestinationDir>$(BUILD_SOURCESDIRECTORY)\artifacts\helixresults</HelixResultsDestinationDir>
-    <WorkItemCommand>$(Python) $(ProductDirectory)\superpmi_diffs.py -base_jit_directory $(ProductDirectory)\base -diff_jit_directory $(ProductDirectory)\diff -log_directory $(SuperpmiLogsLocation)</WorkItemCommand>
-    <WorkItemTimeout>2:00</WorkItemTimeout>
   </PropertyGroup>
 
   <PropertyGroup>
@@ -45,6 +43,17 @@
     <HelixType>$(_HelixType)</HelixType>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <!-- Default to asmdiffs -->
+    <SuperPmiDiffType Condition=" '$(_SuperPmiDiffType)' == '' ">asmdiffs</SuperPmiDiffType>
+    <SuperPmiDiffType Condition=" '$(_SuperPmiDiffType)' != '' ">$(_SuperPmiDiffType)</SuperPmiDiffType>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <WorkItemCommand>$(Python) $(ProductDirectory)\superpmi_diffs.py -type $(SuperPmiDiffType) -base_jit_directory $(ProductDirectory)\base -diff_jit_directory $(ProductDirectory)\diff -log_directory $(SuperpmiLogsLocation)</WorkItemCommand>
+    <WorkItemTimeout>2:00</WorkItemTimeout>
+  </PropertyGroup>
+
   <ItemGroup>
     <HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
       <PayloadDirectory>%(Identity)</PayloadDirectory>
@@ -68,7 +77,9 @@
     <HelixWorkItem Include="@(SPMI_Partition)">
       <Command>$(WorkItemCommand) -arch %(HelixWorkItem.Architecture) -platform %(HelixWorkItem.Platform)</Command>
       <Timeout>$(WorkItemTimeout)</Timeout>
-      <DownloadFilesFromResults>superpmi_asmdiffs_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log;superpmi_tpdiff_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log;superpmi_download_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log;superpmi_diff_summary_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).md;superpmi_tpdiff_summary_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).md;Asmdiffs_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).zip</DownloadFilesFromResults>
+      <DownloadFilesFromResults Condition=" '$(SuperPmiDiffType)'=='asmdiffs' ">superpmi_download_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log;superpmi_asmdiffs_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log;superpmi_asmdiffs_summary_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).md;Asmdiffs_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).zip</DownloadFilesFromResults>
+      <DownloadFilesFromResults Condition=" '$(SuperPmiDiffType)'=='tpdiff' "  >superpmi_download_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log;superpmi_tpdiff_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log;superpmi_tpdiff_summary_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).md</DownloadFilesFromResults>
+      <DownloadFilesFromResults Condition=" '$(SuperPmiDiffType)'=='all' "     >superpmi_download_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log;superpmi_asmdiffs_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log;superpmi_asmdiffs_summary_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).md;Asmdiffs_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).zip;superpmi_tpdiff_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log;superpmi_tpdiff_summary_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).md</DownloadFilesFromResults>
     </HelixWorkItem>
   </ItemGroup>
   </Project>
diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py
@@ -4320,6 +4320,11 @@ def verify_base_diff_args():
                             lambda arch: arch == "x86" or arch == "x64",
                             "Throughput measurements not supported on platform {}".format(coreclr_args.arch))
 
+        coreclr_args.verify(determine_coredis_tools(coreclr_args),
+                            "coredistools_location",
+                            os.path.isfile,
+                            "Unable to find coredistools.")
+
         process_base_jit_path_arg(coreclr_args)
         download_clrjit_pintool(coreclr_args)