python - pr

eng/common/scripts/Package-Properties.ps1

@@ -15,6 +15,7 @@ class PackageProps
     [boolean]$IsNewSdk
     [string]$ArtifactName
     [string]$ReleaseStatus
+    [boolean]$IncludedForValidation
     [string[]]$AdditionalValidationPackages
 
     PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory)
@@ -38,6 +39,7 @@ class PackageProps
         $this.Version = $version
         $this.DirectoryPath = $directoryPath
         $this.ServiceDirectory = $serviceDirectory
+        $this.IncludedForValidation = $false
 
         if (Test-Path (Join-Path $directoryPath "README.md"))
         {
@@ -143,6 +145,8 @@ function Get-PrPkgProperties([string]$InputDiffJson) {
         $key = $addition.Replace($RepoRoot, "").TrimStart('\/')
 
         if ($lookup[$key]) {
+            $lookup[$key].IncludedForValidation = $true
+            Write-Host $lookup[$key].IncludedForValidation
             $packagesWithChanges += $lookup[$key]
         }
     }

eng/pipelines/templates/jobs/ci.tests.yml

@@ -79,9 +79,12 @@ jobs:
       value: ${{ parameters.InjectedPackages }}
 
     steps:
-    - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+    - template: /eng/pipelines/templates/steps/download-package-artifacts.yml
+
+    - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml
       parameters:
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
+        PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
 
     - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml
       parameters:
@@ -112,22 +115,14 @@ jobs:
 
     - template: ../steps/build-test.yml
       parameters:
-        ServiceDirectory: ${{ parameters.ServiceDirectory }}
         TestMarkArgument: ${{ parameters.TestMarkArgument }}
         AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"'
-        CoverageArg: $(CoverageArg)
         PythonVersion: $(PythonVersion)
         ToxTestEnv: $(toxenv)
         ToxEnvParallel: ${{ parameters.ToxEnvParallel }}
         InjectedPackages: $(InjectedPackages)
         TestProxy: ${{ parameters.TestProxy }}
         BeforeTestSteps:
-          - template: /eng/pipelines/templates/steps/download-package-artifacts.yml
-
-          - template: ../steps/set-dev-build.yml
-            parameters:
-              ServiceDirectory: ${{ parameters.ServiceDirectory }}
-
           - ${{ each step in parameters.BeforeTestSteps }}:
             -  ${{ step }}
         AfterTestSteps: ${{ parameters.AfterTestSteps }}

eng/pipelines/templates/jobs/ci.yml

@@ -75,14 +75,14 @@ jobs:
       ComponentDetection.SourcePath: '$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}'
 
     steps:
-    - template: /eng/pipelines/templates/steps/build.yml
+    - template: /eng/pipelines/templates/steps/build-package-artifacts.yml
       parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}
         BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
         TestPipeline: ${{ parameters.TestPipeline }}
-        ArtifactSuffix: linux
         Artifacts: ${{ parameters.Artifacts }}
+        ArtifactSuffix: linux
+        BuildTargetingString: ${{ parameters.BuildTargetingString }}
 
   - job: 'Build_Windows'
     timeoutInMinutes: 90
@@ -96,14 +96,14 @@ jobs:
       ComponentDetection.SourcePath: '$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}'
 
     steps:
-    - template: /eng/pipelines/templates/steps/build.yml
+    - template: /eng/pipelines/templates/steps/build-package-artifacts.yml
       parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}
         BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
         TestPipeline: ${{ parameters.TestPipeline }}
-        ArtifactSuffix: windows
         Artifacts: ${{ parameters.Artifacts }}
+        ArtifactSuffix: windows
+        BuildTargetingString: ${{ parameters.BuildTargetingString }}
 
   - job: 'Build_MacOS'
     timeoutInMinutes: 90
@@ -117,14 +117,14 @@ jobs:
       ComponentDetection.SourcePath: '$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}'
 
     steps:
-    - template: /eng/pipelines/templates/steps/build.yml
+    - template: /eng/pipelines/templates/steps/build-package-artifacts.yml
       parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}
         BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
         TestPipeline: ${{ parameters.TestPipeline }}
-        ArtifactSuffix: mac
         Artifacts: ${{ parameters.Artifacts }}
+        ArtifactSuffix: mac
+        BuildTargetingString: ${{ parameters.BuildTargetingString }}
 
   - job: 'Build_Extended'
     displayName: Build Extended
@@ -146,9 +146,41 @@ jobs:
     steps:
     - template: /eng/pipelines/templates/steps/download-package-artifacts.yml
 
-    - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+    - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml
       parameters:
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
+        PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
+
+    - template: ../steps/build-extended-artifacts.yml
+      parameters:
+        ServiceDirectory: ${{ parameters.ServiceDirectory }}
+        BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
+        BuildDocs: false
+        RunApiStubGen: true
+        TestPipeline: ${{ parameters.TestPipeline }}
+        Artifacts: ${{ parameters.Artifacts }}
+
+  - job: 'Build_Documentation'
+    displayName: Build Docs
+    dependsOn:
+      - 'Build_Linux'
+      - 'Build_Windows'
+      - 'Build_MacOS'
+
+    timeoutInMinutes: 90
+
+    pool:
+      name: $(LINUXPOOL)
+      image: $(LINUXVMIMAGE)
+      os: linux
+
+    steps:
+    - template: /eng/pipelines/templates/steps/download-package-artifacts.yml
+
+    - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml
+      parameters:
+        BuildTargetingString: ${{ parameters.BuildTargetingString }}
+        PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
 
     - template: ../steps/build-extended-artifacts.yml
       parameters:
@@ -157,6 +189,7 @@ jobs:
         BuildDocs: ${{ parameters.BuildDocs }}
         TestPipeline: ${{ parameters.TestPipeline }}
         Artifacts: ${{ parameters.Artifacts }}
+        RunApiStubGen: false
 
   - job: 'Analyze'
     condition: and(succeeded(), ne(variables['Skip.Analyze'], 'true'))
@@ -172,10 +205,6 @@ jobs:
       os: linux
 
     steps:
-    - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
-      parameters:
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}
-
     - template: /eng/common/pipelines/templates/steps/check-spelling.yml
       parameters:
         ContinueOnError: false
@@ -196,6 +225,11 @@ jobs:
         artifactName: 'packages_extended'
         targetPath: $(Build.ArtifactStagingDirectory)
 
+    - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml
+      parameters:
+        BuildTargetingString: ${{ parameters.BuildTargetingString }}
+        PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
+
     - template: ../steps/analyze.yml
       parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
@@ -219,6 +253,21 @@ jobs:
       MatrixConfigs: ${{ parameters.MatrixConfigs }}
       MatrixFilters: ${{ parameters.MatrixFilters }}
       MatrixReplace: ${{ parameters.MatrixReplace }}
+      PreGenerationSteps:
+
+        - template: /eng/pipelines/templates/steps/download-package-artifacts.yml
+          condition: eq(variables['Build.Reason'], 'PullRequest')
+
+        - task: Powershell@2
+          inputs:
+            pwsh: true
+            filePath: eng/scripts/distribute-packages-to-matrix.ps1
+            arguments: >-
+              -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo"
+              -PlatformMatrix "${{ parameters.MatrixConfigs[0].Path }}"
+          displayName: 'Distribute Packages to Matrix'
+          condition: eq(variables['Build.Reason'], 'PullRequest')
+
       CloudConfig:
         Cloud: Public
       AdditionalParameters:
@@ -253,15 +302,12 @@ jobs:
         - pwsh: |
             python -m pip install "./tools/azure-sdk-tools[build]"
           displayName: 'Prep Environment'
-        - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
-          parameters:
-            BuildTargetingString: ${{ parameters.BuildTargetingString }}
         - task: PythonScript@0
           displayName: 'Ensure service coverage'
           inputs:
             scriptPath: '$(Build.SourcesDirectory)/scripts/devops_tasks/update_regression_services.py'
             arguments: >-
-              "$(TargetingString)"
+              "azure*"
               --service="${{ parameters.ServiceDirectory }}"
               --json=$(Build.SourcesDirectory)/eng/pipelines/templates/stages/regression-job-matrix.json
       CloudConfig:

eng/pipelines/templates/jobs/live.tests.yml

@@ -110,7 +110,7 @@ jobs:
         parameters:
           AgentImage: ${{ parameters.OSName }}
 
-      - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+      - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml
         parameters:
           BuildTargetingString: ${{ parameters.BuildTargetingString }}
 

eng/pipelines/templates/jobs/regression.yml

@@ -71,7 +71,7 @@ jobs:
 
     # Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`.
     steps:
-      - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+      - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml
         parameters:
           BuildTargetingString: ${{ parameters.BuildTargetingString }}
 

eng/pipelines/templates/stages/platform-matrix.json

@@ -10,32 +10,8 @@
       "ubuntu-20.04": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" },
       "macos-latest": { "OSVmImage": "env:MACVMIMAGE", "Pool": "env:MACPOOL" }
     },
-    "PythonVersion": [ "3.8", "pypy3.9", "3.11", "3.10" ],
-    "CoverageArg": "--disablecov",
-    "TestSamples": "false"
-  },
-  "include": [
-    {
-      "CoverageConfig": {
-        "ubuntu2004_39_coverage": {
-          "OSVmImage": "env:LINUXVMIMAGE",
-          "Pool": "env:LINUXPOOL",
-          "PythonVersion": "3.9",
-          "CoverageArg": "",
-          "TestSamples": "false"
-        }
-      }
-    },
-    {
-      "Config": {
-        "Ubuntu2004_312": {
-          "OSVmImage": "env:LINUXVMIMAGE",
-          "Pool": "env:LINUXPOOL",
-          "PythonVersion": "3.12",
-          "CoverageArg": "--disablecov",
-          "TestSamples": "false"
-        }
-      }
-    }
-  ]
+    "PythonVersion": [ "3.8", "pypy3.9", "3.9", "3.11", "3.12", "3.10" ],
+    "TestSamples": "false",
+    "TargetingString": "azure*"
+  }
 }

eng/pipelines/templates/steps/analyze.yml

@@ -8,12 +8,7 @@ parameters:
   ValidateFormatting: false
   GenerateApiReviewForManualOnly: false
 
-# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files:
-#     eng/pipelines/templates/jobs/ci.tests.yml
-#     eng/pipelines/templates/jobs/ci.yml
-#     eng/pipelines/templates/jobs/live.test.yml
-
-# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`.
+# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml.
 steps:
   - template: /eng/pipelines/templates/steps/analyze_dependency.yml
     parameters:
@@ -32,14 +27,20 @@ steps:
       scriptPath: 'scripts/devops_tasks/set_tox_environment.py'
       arguments: '"$(TargetingString)" --team-project="$(System.TeamProject)" --service="${{ parameters.ServiceDirectory }}"'
 
-  - ${{ each artifact in parameters.Artifacts }}:
-    - ${{if ne(artifact.skipVerifyChangeLog, 'true')}}:
-      - template: /eng/common/pipelines/templates/steps/verify-changelog.yml
-        parameters:
-          PackageName: ${{artifact.name}}
-          ServiceName: ${{parameters.ServiceDirectory}}
-          ForRelease: false
-          Condition: succeededOrFailed()
+  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+    - template: /eng/common/pipelines/templates/steps/verify-changelogs.yml
+      parameters:
+        PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
+        Condition: succeededOrFailed()
+  - ${{ else }}:
+    - ${{ each artifact in parameters.Artifacts }}:
+      - ${{if ne(artifact.skipVerifyChangeLog, 'true')}}:
+        - template: /eng/common/pipelines/templates/steps/verify-changelog.yml
+          parameters:
+            PackageName: ${{artifact.name}}
+            ServiceName: ${{parameters.ServiceDirectory}}
+            ForRelease: false
+            Condition: succeededOrFailed()
 
   - script: |
       python -m pip install "./tools/azure-sdk-tools[build]" -q -I

eng/pipelines/templates/steps/analyze_dependency.yml

@@ -13,9 +13,9 @@ steps:
     displayName: 'Install Python Tools'
     condition: succeededOrFailed()
 
-  - template: /eng/common/pipelines/templates/steps/verify-readme.yml
+  - template: /eng/common/pipelines/templates/steps/verify-readmes.yml
     parameters:
-      ScanPath: ${{ parameters.ScanPath }}
+      PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
       Condition: succeededOrFailed()
 
   - pwsh: |