From 6492b585e417fa7ddcb28ee71bfed2276af70201 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Fri, 8 Mar 2024 15:05:58 -0800 Subject: [PATCH 1/3] Squashed down changes converting all ci.yml pipelines to 1ES templates --- eng/pipelines/templates/jobs/ci.tests.yml | 22 +- eng/pipelines/templates/jobs/ci.yml | 27 +- eng/pipelines/templates/jobs/live.tests.yml | 24 +- .../templates/stages/archetype-js-release.yml | 589 +++++++++--------- .../templates/stages/archetype-sdk-client.yml | 107 ++-- .../stages/archetype-sdk-tests-isolated.yml | 122 ++++ .../templates/stages/archetype-sdk-tests.yml | 116 ++-- .../templates/stages/cosmos-sdk-client.yml | 140 +++-- .../templates/stages/platform-matrix.json | 25 +- eng/pipelines/templates/steps/analyze.yml | 15 +- eng/pipelines/templates/steps/build.yml | 13 +- eng/pipelines/templates/steps/test.yml | 3 +- eng/pipelines/templates/variables/globals.yml | 1 - eng/pipelines/templates/variables/image.yml | 26 + .../phone-numbers-livetest-matrix.json | 20 +- .../keyvault-admin/platform-matrix.json | 4 +- .../platform-matrix.json | 4 +- .../keyvault-keys/platform-matrix.json | 8 +- .../keyvault-secrets/platform-matrix.json | 4 +- sdk/template/template/tests.yml | 21 +- 20 files changed, 747 insertions(+), 544 deletions(-) create mode 100644 eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml create mode 100644 eng/pipelines/templates/variables/image.yml diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 8a35f5c7b7c3..c193f5902ea8 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -11,14 +11,17 @@ parameters: - name: Matrix type: string - name: DependsOn - type: string - default: '' + type: object + default: [] - name: UsePlatformContainer type: boolean default: false - name: CloudConfig type: object default: {} + - name: OSName + type: string + default: '' jobs: - job: @@ -39,10 +42,16 @@ jobs: pool: name: $(Pool) - vmImage: $(OSVmImage) - ${{ if eq(parameters.UsePlatformContainer, 'true') }}: - # Add a default so the job doesn't fail when the matrix is empty - container: $[ variables['Container'] ] + # 1es pipeline templates converts `image` to demands: ImageOverride under the hood + # which is incompatible with image selection in the default non-1es hosted pools + ${{ if eq(parameters.OSName, 'macOS') }}: + vmImage: $(OSVmImage) + ${{ else }}: + image: $(OSVmImage) + os: ${{ parameters.OSName }} + ${{ if eq(parameters.UsePlatformContainer, 'true') }}: + # Add a default so the job doesn't fail when the matrix is empty + container: $[ variables['Container'] ] variables: - template: ../variables/globals.yml @@ -55,3 +64,4 @@ jobs: Artifacts: ${{ parameters.Artifacts }} ServiceDirectory: ${{ parameters.ServiceDirectory }} TestProxy: ${{ parameters.TestProxy }} + OSName: ${{ parameters.OSName }} diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 9ce550da79be..2d91faa403f0 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -28,14 +28,11 @@ parameters: jobs: - job: "Build" - variables: - Codeql.Enabled: true - Codeql.BuildIdentifier: ${{ parameters.ServiceDirectory }} - Codeql.SkipTaskAutoInjection: false pool: - name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux steps: - script: | @@ -55,8 +52,9 @@ jobs: - job: "Analyze" pool: - name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux steps: - template: ../steps/common.yml @@ -67,19 +65,12 @@ jobs: ServiceDirectory: ${{ parameters.ServiceDirectory }} TestPipeline: ${{ parameters.TestPipeline }} - - job: Compliance - pool: - name: azsdk-pool-mms-win-2022-general - vmImage: MMS2022 - steps: - - template: /eng/common/pipelines/templates/steps/credscan.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - - ${{ if ne(parameters.RunUnitTests, false) }}: - - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml + - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml parameters: JobTemplatePath: /eng/pipelines/templates/jobs/ci.tests.yml + OsVmImage: $(LINUXVMIMAGE) + Pool: $(LINUXPOOL) MatrixConfigs: ${{ parameters.MatrixConfigs }} MatrixFilters: ${{ parameters.MatrixFilters }} MatrixReplace: ${{ parameters.MatrixReplace }} diff --git a/eng/pipelines/templates/jobs/live.tests.yml b/eng/pipelines/templates/jobs/live.tests.yml index e9e4408f6701..f5aeb7428bd8 100644 --- a/eng/pipelines/templates/jobs/live.tests.yml +++ b/eng/pipelines/templates/jobs/live.tests.yml @@ -40,7 +40,8 @@ parameters: - name: UsePlatformContainer type: boolean default: false - +- name: OSName + type: string jobs: - job: @@ -57,7 +58,13 @@ jobs: pool: name: $(Pool) - vmImage: $(OSVmImage) + # 1es pipeline templates converts `image` to demands: ImageOverride under the hood + # which is incompatible with image selection in the default non-1es hosted pools + ${{ if eq(parameters.OSName, 'macOS') }}: + vmImage: $(OSVmImage) + ${{ else }}: + image: $(OSVmImage) + os: ${{ parameters.OSName }} timeoutInMinutes: ${{ parameters.TimeoutInMinutes }} @@ -190,13 +197,12 @@ jobs: codeCoverageTool: Cobertura summaryFileLocation: "$(PackagePath)/coverage/cobertura-coverage.xml" - - task: PublishPipelineArtifact@1 - displayName: "Publish Browser Code Coverage Report Artifact" - continueOnError: true - condition: and(succeededOrFailed(),eq(variables['TestType'], 'browser'),eq(variables['PublishCodeCoverage'], true)) - inputs: - path: "$(PackagePath)/coverage-browser" - artifact: BrowserCodeCoverageReport + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: "$(PackagePath)/coverage-browser" + ArtifactName: BrowserCodeCoverageReport + CustomCondition: and(succeededOrFailed(),eq(variables['TestType'], 'browser'),eq(variables['PublishCodeCoverage'], true)) + SbomEnabled: false # Unlink node_modules folders to significantly improve performance of subsequent tasks # which need to walk the directory tree (and are hardcoded to follow symlinks). diff --git a/eng/pipelines/templates/stages/archetype-js-release.yml b/eng/pipelines/templates/stages/archetype-js-release.yml index d14451c8cdbc..50eb0868d942 100644 --- a/eng/pipelines/templates/stages/archetype-js-release.yml +++ b/eng/pipelines/templates/stages/archetype-js-release.yml @@ -1,319 +1,298 @@ parameters: Artifacts: [] TestPipeline: false - ArtifactName: 'not-specified' + ArtifactName: not-specified DependsOn: Build - Registry: 'https://registry.npmjs.org/' - PrivateRegistry: 'https://pkgs.dev.azure.com/azure-sdk/internal/_packaging/azure-sdk-for-js-pr/npm/registry/' + Registry: https://registry.npmjs.org/ + PrivateRegistry: https://pkgs.dev.azure.com/azure-sdk/internal/_packaging/azure-sdk-for-js-pr/npm/registry/ TargetDocRepoOwner: '' TargetDocRepoName: '' ServiceDirectory: '' + stages: - ${{if and(in(variables['Build.Reason'], 'Manual', ''), eq(variables['System.TeamProject'], 'internal'))}}: - - ${{ each artifact in parameters.Artifacts }}: - - stage: - variables: - - template: /eng/pipelines/templates/variables/globals.yml - displayName: 'Release: ${{artifact.name}}' - dependsOn: ${{parameters.DependsOn}} - condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-js-pr')) - jobs: - - deployment: TagRepository - displayName: "Create release tag" - condition: ne(variables['Skip.TagRepository'], 'true') - environment: github - - pool: - name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 - - strategy: - runOnce: - deploy: - steps: - - checkout: self - - template: /eng/common/pipelines/templates/steps/retain-run.yml - - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml - parameters: - PackageName: "@azure/template" - ServiceDirectory: "template" - TestPipeline: ${{ parameters.TestPipeline }} - - template: /eng/common/pipelines/templates/steps/verify-changelog.yml - parameters: - PackageName: ${{artifact.name}} - ServiceName: ${{parameters.ServiceDirectory}} - ForRelease: true - - template: /eng/common/pipelines/templates/steps/verify-restapi-spec-location.yml - parameters: - PackageName: ${{artifact.name}} - ServiceDirectory: ${{parameters.ServiceDirectory}} - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}} - - pwsh: | - Get-ChildItem -Recurse ${{parameters.ArtifactName}}/${{artifact.name}} - workingDirectory: $(Pipeline.Workspace) - displayName: Output Visible Artifacts - - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml - parameters: - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} - PackageRepository: Npm - ReleaseSha: $(Build.SourceVersion) - RepoId: Azure/azure-sdk-for-js - WorkingDirectory: $(System.DefaultWorkingDirectory) - - - ${{if ne(artifact.skipPublishPackage, 'true')}}: - - deployment: PublishPackage - displayName: "Publish to npmjs" - condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) - environment: npm - dependsOn: TagRepository - - pool: - name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 - - strategy: - runOnce: - deploy: - steps: - - checkout: self - - script: | - export DETECTED_PACKAGE_NAME=`ls $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.tgz` - echo "##vso[task.setvariable variable=Package.Archive]$DETECTED_PACKAGE_NAME" - displayName: Detecting package archive - - - pwsh: | - write-host "$(Package.Archive)" - $result = eng/scripts/get-npm-tags.ps1 -packageArtifact $(Package.Archive) -workingDirectory $(System.DefaultWorkingDirectory)/temp - write-host "Tag: $($result.Tag)" - write-host "Additional tag: $($result.AdditionalTag)" - echo "##vso[task.setvariable variable=Tag]$($result.Tag)" - echo "##vso[task.setvariable variable=AdditionalTag]$($result.AdditionalTag)" - condition: and(succeeded(), ne(variables['Skip.AutoAddTag'], 'true')) - displayName: 'Set Tag and Additional Tag' - - - script: | - npm install $(Package.Archive) - displayName: 'Validating package can be installed' - condition: succeeded() - - - task: PowerShell@2 - displayName: 'Publish to npmjs.org' - inputs: - targetType: filePath - filePath: "eng/tools/publish-to-npm.ps1" - arguments: '-pathToArtifacts $(Package.Archive) -accessLevel "public" -tag "$(Tag)" -additionalTag "$(AdditionalTag)" -registry ${{parameters.Registry}} -npmToken $(azure-sdk-npm-token)' - pwsh: true - condition: succeeded() - - - pwsh: | - write-host "$(Package.Archive)" - eng/scripts/cleanup-npm-next-tag.ps1 -packageArtifact $(Package.Archive) -workingDirectory $(System.DefaultWorkingDirectory)/temp -npmToken $(azure-sdk-npm-token) - displayName: 'Cleanup Npm Next Tag' - condition: and(succeeded(), ne(variables['Skip.RemoveOldTag'], 'true')) - - - - ${{if ne(artifact.skipPublishDocMs, 'true')}}: - - deployment: PublishDocs - displayName: Docs.MS Release - condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) - environment: githubio - dependsOn: PublishPackage - - pool: - name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 - - strategy: - runOnce: - deploy: - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - sdk/**/*.md - - .github/CODEOWNERS - - download: current - - - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml - - - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml - parameters: - PackageInfoLocations: - - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json - RepoId: Azure/azure-sdk-for-js - WorkingDirectory: $(System.DefaultWorkingDirectory) - TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} - TargetDocRepoName: ${{parameters.TargetDocRepoName}} - Language: 'javascript' - SparseCheckoutPaths: - - docs-ref-services/ - - metadata/ - - ci-configs/packages-latest.json - - ci-configs/packages-preview.json - - - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: - - deployment: PublishDocsGitHubIO - displayName: Publish Docs to GitHubIO Blob Storage - condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) - environment: githubio - dependsOn: PublishPackage - - pool: - name: azsdk-pool-mms-win-2022-general - vmImage: MMS2022 - - strategy: - runOnce: - deploy: - steps: - - checkout: self - - pwsh: | - Get-ChildItem -Recurse ${{parameters.ArtifactName}}/${{artifact.name}} - workingDirectory: $(Pipeline.Workspace) - displayName: Output Visible Artifacts - - template: /eng/common/pipelines/templates/steps/publish-blobs.yml - parameters: - FolderForUpload: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}' - BlobSASKey: '$(azure-sdk-docs-prod-sas)' - BlobName: '$(azure-sdk-docs-prod-blob-name)' - TargetLanguage: 'javascript' - ArtifactLocation: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}' - # we override the regular script path because we have cloned the build tools repo as a separate artifact. - ScriptPath: 'eng/common/scripts/copy-docs-to-blobstorage.ps1' - - - ${{if ne(artifact.skipUpdatePackageVersion, 'true')}}: - - deployment: UpdatePackageVersion - displayName: "Update Package Version" - condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) - environment: github - dependsOn: PublishPackage - - pool: - name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 - - strategy: - runOnce: - deploy: - steps: - - checkout: self - - - template: /eng/pipelines/templates/steps/common.yml - - - bash: | - npm install - workingDirectory: ./eng/tools/versioning - displayName: Install versioning tool dependencies - - - bash: | - node ./eng/tools/versioning/increment.js --artifact-name ${{ artifact.name }} --repo-root . - displayName: Increment package version - - - bash: | - node common/scripts/install-run-rush.js install - displayName: "Install dependencies" - - # Disabled until packages can be updated to support ES2019 syntax. - # - bash: | - # npm install -g ./common/tools/dev-tool - # npm install ./eng/tools/eng-package-utils - # node ./eng/tools/eng-package-utils/update-samples.js ${{ artifact.name }} - # displayName: Update samples - - - template: /eng/common/pipelines/templates/steps/create-pull-request.yml - parameters: - RepoName: azure-sdk-for-js - PRBranchName: post-release-automation-${{ parameters.ServiceDirectory }}-$(Build.BuildId) - CommitMsg: "Post release automated changes for ${{ artifact.name }}" - PRTitle: "Post release automated changes for ${{ parameters.ServiceDirectory }} releases" - CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' - - + - ${{ each artifact in parameters.Artifacts }}: + - stage: + variables: + - template: /eng/pipelines/templates/variables/globals.yml + displayName: 'Release: ${{artifact.name}}' + dependsOn: ${{parameters.DependsOn}} + condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-js-pr')) + jobs: + - deployment: TagRepository + displayName: Create release tag + condition: ne(variables['Skip.TagRepository'], 'true') + environment: github + pool: + name: azsdk-pool-mms-ubuntu-2004-general + image: azsdk-pool-mms-ubuntu-2004-1espt + os: linux + strategy: + runOnce: + deploy: + steps: + - checkout: self + - template: /eng/common/pipelines/templates/steps/retain-run.yml + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + parameters: + PackageName: '@azure/template' + ServiceDirectory: template + TestPipeline: ${{ parameters.TestPipeline }} + - template: /eng/common/pipelines/templates/steps/verify-changelog.yml + parameters: + PackageName: ${{artifact.name}} + ServiceName: ${{parameters.ServiceDirectory}} + ForRelease: true + - template: /eng/common/pipelines/templates/steps/verify-restapi-spec-location.yml + parameters: + PackageName: ${{artifact.name}} + ServiceDirectory: ${{parameters.ServiceDirectory}} + ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}} + - pwsh: > + Get-ChildItem -Recurse ${{parameters.ArtifactName}}/${{artifact.name}} + workingDirectory: $(Pipeline.Workspace) + displayName: Output Visible Artifacts + - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml + parameters: + ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} + PackageRepository: Npm + ReleaseSha: $(Build.SourceVersion) + RepoId: Azure/azure-sdk-for-js + WorkingDirectory: $(System.DefaultWorkingDirectory) + - ${{if ne(artifact.skipPublishPackage, 'true')}}: + - deployment: PublishPackage + displayName: Publish to npmjs + condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) + environment: npm + dependsOn: TagRepository + pool: + name: azsdk-pool-mms-ubuntu-2004-general + image: azsdk-pool-mms-ubuntu-2004-1espt + os: linux + strategy: + runOnce: + deploy: + steps: + - checkout: self + - script: > + export DETECTED_PACKAGE_NAME=`ls $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.tgz` + + echo "##vso[task.setvariable variable=Package.Archive]$DETECTED_PACKAGE_NAME" + displayName: Detecting package archive + - pwsh: > + write-host "$(Package.Archive)" + + $result = eng/scripts/get-npm-tags.ps1 -packageArtifact $(Package.Archive) -workingDirectory $(System.DefaultWorkingDirectory)/temp + + write-host "Tag: $($result.Tag)" + + write-host "Additional tag: $($result.AdditionalTag)" + + echo "##vso[task.setvariable variable=Tag]$($result.Tag)" + + echo "##vso[task.setvariable variable=AdditionalTag]$($result.AdditionalTag)" + condition: and(succeeded(), ne(variables['Skip.AutoAddTag'], 'true')) + displayName: Set Tag and Additional Tag + - script: > + npm install $(Package.Archive) + displayName: Validating package can be installed + condition: succeeded() + - task: PowerShell@2 + displayName: Publish to npmjs.org + inputs: + targetType: filePath + filePath: eng/tools/publish-to-npm.ps1 + arguments: -pathToArtifacts $(Package.Archive) -accessLevel "public" -tag "$(Tag)" -additionalTag "$(AdditionalTag)" -registry ${{parameters.Registry}} -npmToken $(azure-sdk-npm-token) + pwsh: true + condition: succeeded() + - pwsh: > + write-host "$(Package.Archive)" + + eng/scripts/cleanup-npm-next-tag.ps1 -packageArtifact $(Package.Archive) -workingDirectory $(System.DefaultWorkingDirectory)/temp -npmToken $(azure-sdk-npm-token) + displayName: Cleanup Npm Next Tag + condition: and(succeeded(), ne(variables['Skip.RemoveOldTag'], 'true')) + - ${{if ne(artifact.skipPublishDocMs, 'true')}}: + - deployment: PublishDocs + displayName: Docs.MS Release + condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) + environment: githubio + dependsOn: PublishPackage + pool: + name: azsdk-pool-mms-ubuntu-2004-general + image: azsdk-pool-mms-ubuntu-2004-1espt + os: linux + strategy: + runOnce: + deploy: + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - sdk/**/*.md + - .github/CODEOWNERS + - download: current + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml + parameters: + PackageInfoLocations: + - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json + RepoId: Azure/azure-sdk-for-js + WorkingDirectory: $(System.DefaultWorkingDirectory) + TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} + TargetDocRepoName: ${{parameters.TargetDocRepoName}} + Language: javascript + SparseCheckoutPaths: + - docs-ref-services/ + - metadata/ + - ci-configs/packages-latest.json + - ci-configs/packages-preview.json + - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: + - deployment: PublishDocsGitHubIO + displayName: Publish Docs to GitHubIO Blob Storage + condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) + environment: githubio + dependsOn: PublishPackage + pool: + name: azsdk-pool-mms-win-2022-general + image: azsdk-pool-mms-win-2022-1espt + os: windows + strategy: + runOnce: + deploy: + steps: + - checkout: self + - pwsh: > + Get-ChildItem -Recurse ${{parameters.ArtifactName}}/${{artifact.name}} + workingDirectory: $(Pipeline.Workspace) + displayName: Output Visible Artifacts + - template: /eng/common/pipelines/templates/steps/publish-blobs.yml + parameters: + FolderForUpload: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} + BlobSASKey: $(azure-sdk-docs-prod-sas) + BlobName: $(azure-sdk-docs-prod-blob-name) + TargetLanguage: javascript + ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} + ScriptPath: eng/common/scripts/copy-docs-to-blobstorage.ps1 + - ${{if ne(artifact.skipUpdatePackageVersion, 'true')}}: + - deployment: UpdatePackageVersion + displayName: Update Package Version + condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) + environment: github + dependsOn: PublishPackage + pool: + name: azsdk-pool-mms-ubuntu-2004-general + image: azsdk-pool-mms-ubuntu-2004-1espt + os: linux + strategy: + runOnce: + deploy: + steps: + - checkout: self + - template: /eng/pipelines/templates/steps/common.yml + - bash: > + npm install + workingDirectory: ./eng/tools/versioning + displayName: Install versioning tool dependencies + - bash: > + node ./eng/tools/versioning/increment.js --artifact-name ${{ artifact.name }} --repo-root . + displayName: Increment package version + - bash: > + node common/scripts/install-run-rush.js install + displayName: Install dependencies + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + RepoName: azure-sdk-for-js + PRBranchName: post-release-automation-${{ parameters.ServiceDirectory }}-$(Build.BuildId) + CommitMsg: Post release automated changes for ${{ artifact.name }} + PRTitle: Post release automated changes for ${{ parameters.ServiceDirectory }} releases + CloseAfterOpenForTesting: ${{ parameters.TestPipeline }} - stage: Integration dependsOn: ${{parameters.DependsOn}} variables: - - template: /eng/pipelines/templates/variables/globals.yml + - template: /eng/pipelines/templates/variables/globals.yml jobs: - - job: PublishPackages - # Run Integration job only if SetDevVersion is set to true or ( SetDevVersion is empty and job is a scheduled CI run) - # If SetDevVersion is set to false then we should skip integration job even for scheduled runs. - condition: or(eq(variables['SetDevVersion'], 'true'), and(eq(variables['Build.Reason'],'Schedule'), eq(variables['System.TeamProject'], 'internal'))) - displayName: Publish package to daily feed - pool: - name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 - steps: - - checkout: self - - download: current - artifact: ${{parameters.ArtifactName}} - timeoutInMinutes: 5 - - ${{ each artifact in parameters.Artifacts }}: - - ${{if ne(artifact.skipPublishDevFeed, 'true')}}: - - pwsh: | - $detectedPackageName=Get-ChildItem $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.tgz - Write-Host "Detected package name: $($detectedPackageName)" - if ($detectedPackageName -notmatch "-alpha") - { - Write-Error "Found non alpha version artifact to publish as dev version. Failing publish step. VersionPolicy should be client or core in rush.json to get alpha version build." - exit 1 - } - echo "##vso[task.setvariable variable=Package.Archive]$detectedPackageName" - if ('$(Build.Repository.Name)' -eq 'Azure/azure-sdk-for-js') { - $npmToken="$(azure-sdk-npm-token)" - $registry="${{parameters.Registry}}" - } - else { - $npmToken="$(azure-sdk-devops-npm-token)" - $registry="${{parameters.PrivateRegistry}}" - } - echo "##vso[task.setvariable variable=NpmToken]$npmToken" - echo "##vso[task.setvariable variable=Registry]$registry" - displayName: Detecting package archive_${{artifact.name}} - - - task: PowerShell@2 - displayName: "Publish_${{artifact.name}} to dev feed" - inputs: - targetType: filePath - filePath: "eng/tools/publish-to-npm.ps1" - arguments: '-pathToArtifacts $(Package.Archive) -accessLevel "public" -tag "dev" -registry "$(Registry)" -npmToken "$(NpmToken)"' - - - job: PublishDocsToNightlyBranch - condition: or(eq(variables['SetDevVersion'], 'true'), and(eq(variables['Build.Reason'],'Schedule'), eq(variables['System.TeamProject'], 'internal'), ne(variables['Skip.PublishDocs'], 'true'))) - dependsOn: PublishPackages - pool: - name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 - - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - sdk/**/*.md - - .github/CODEOWNERS - - download: current - - pwsh: | - Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/ - displayName: Show visible artifacts - - - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml - - - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml - parameters: - PackageInfoLocations: - - ${{ each artifact in parameters.Artifacts }}: - - ${{if ne(artifact.skipPublishDocMs, 'true')}}: - - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json - RepoId: Azure/azure-sdk-for-js - WorkingDirectory: $(System.DefaultWorkingDirectory) - TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} - TargetDocRepoName: ${{parameters.TargetDocRepoName}} - Language: 'javascript' - DailyDocsBuild: true - SparseCheckoutPaths: - - docs-ref-services/ - - metadata/ - - ci-configs/packages-latest.json - - ci-configs/packages-preview.json - - - template: /eng/common/pipelines/templates/steps/docsms-ensure-validation.yml + - job: PublishPackages + condition: or(eq(variables['SetDevVersion'], 'true'), and(eq(variables['Build.Reason'],'Schedule'), eq(variables['System.TeamProject'], 'internal'))) + displayName: Publish package to daily feed + pool: + name: azsdk-pool-mms-ubuntu-2004-general + image: azsdk-pool-mms-ubuntu-2004-1espt + os: linux + steps: + - checkout: self + - download: current + artifact: ${{parameters.ArtifactName}} + timeoutInMinutes: 5 + - ${{ each artifact in parameters.Artifacts }}: + - ${{if ne(artifact.skipPublishDevFeed, 'true')}}: + - pwsh: > + $detectedPackageName=Get-ChildItem $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.tgz + + Write-Host "Detected package name: $($detectedPackageName)" + + if ($detectedPackageName -notmatch "-alpha") + + { + Write-Error "Found non alpha version artifact to publish as dev version. Failing publish step. VersionPolicy should be client or core in rush.json to get alpha version build." + exit 1 + } + + echo "##vso[task.setvariable variable=Package.Archive]$detectedPackageName" + + if ('$(Build.Repository.Name)' -eq 'Azure/azure-sdk-for-js') { + $npmToken="$(azure-sdk-npm-token)" + $registry=${{parameters.Registry}}" + } + + else { + $npmToken="$(azure-sdk-devops-npm-token)" + $registry=${{parameters.PrivateRegistry}}" + } + + echo "##vso[task.setvariable variable=NpmToken]$npmToken" + + echo "##vso[task.setvariable variable=Registry]$registry" + displayName: Detecting package archive_${{artifact.name}} + - task: PowerShell@2 + displayName: Publish_${{artifact.name}} to dev feed + inputs: + targetType: filePath + filePath: eng/tools/publish-to-npm.ps1 + arguments: -pathToArtifacts $(Package.Archive) -accessLevel "public" -tag "dev" -registry "$(Registry)" -npmToken "$(NpmToken)" + - job: PublishDocsToNightlyBranch + condition: or(eq(variables['SetDevVersion'], 'true'), and(eq(variables['Build.Reason'],'Schedule'), eq(variables['System.TeamProject'], 'internal'), ne(variables['Skip.PublishDocs'], 'true'))) + dependsOn: PublishPackages + pool: + name: azsdk-pool-mms-ubuntu-2004-general + vmImage: azsdk-pool-mms-ubuntu-2004-1espt + os: linux + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - sdk/**/*.md + - .github/CODEOWNERS + - download: current + - pwsh: > + Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/ + displayName: Show visible artifacts + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml + parameters: + PackageInfoLocations: + - ${{ each artifact in parameters.Artifacts }}: + - ${{if ne(artifact.skipPublishDocMs, 'true')}}: + - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json + RepoId: Azure/azure-sdk-for-js + WorkingDirectory: $(System.DefaultWorkingDirectory) + TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} + TargetDocRepoName: ${{parameters.TargetDocRepoName}} + Language: javascript + DailyDocsBuild: true + SparseCheckoutPaths: + - docs-ref-services/ + - metadata/ + - ci-configs/packages-latest.json + - ci-configs/packages-preview.json + - template: /eng/common/pipelines/templates/steps/docsms-ensure-validation.yml diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 823aafc85745..1811aba459a4 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -1,3 +1,10 @@ +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + parameters: - name: Artifacts type: object @@ -40,43 +47,65 @@ parameters: type: object default: [] -variables: - - template: ../variables/globals.yml - -stages: - - stage: Build - jobs: - - template: /eng/pipelines/templates/jobs/ci.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - TestProxy: ${{ parameters.TestProxy }} - Artifacts: ${{ parameters.Artifacts }} - ${{ if eq(parameters.ServiceDirectory, 'template') }}: - TestPipeline: true - RunUnitTests: ${{ parameters.RunUnitTests }} - MatrixConfigs: - - ${{ each config in parameters.MatrixConfigs }}: - - ${{ config }} - - ${{ each config in parameters.AdditionalMatrixConfigs }}: - - ${{ config }} - MatrixFilters: - - TestType=node|browser - - DependencyVersion=^$ - - ${{ each filter in parameters.MatrixFilters }}: - - ${{ filter}} - MatrixReplace: ${{ parameters.MatrixReplace }} - IncludeRelease: ${{ parameters.IncludeRelease }} +extends: + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + ${{ else }}: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + settings: + skipBuildTagsForGitHubPullRequests: true + sdl: + sourceAnalysisPool: + name: azsdk-pool-mms-win-2022-general + image: azsdk-pool-mms-win-2022-1espt + os: windows + eslint: + enabled: false + justificationForDisabling: 'ESLint injected task has failures because it uses an old version of mkdirp. We should not fail for tools not controlled by the repo. See: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=3499746' + psscriptanalyzer: + compiled: true + break: true + policy: M365 + credscan: + suppressionsFile: $(Build.SourcesDirectory)/eng/CredScanSuppression.json + toolVersion: 2.3.12.23 + stages: + - stage: Build + jobs: + - template: /eng/pipelines/templates/jobs/ci.yml@self + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + TestProxy: ${{ parameters.TestProxy }} + Artifacts: ${{ parameters.Artifacts }} + ${{ if eq(parameters.ServiceDirectory, 'template') }}: + TestPipeline: true + RunUnitTests: ${{ parameters.RunUnitTests }} + MatrixConfigs: + - ${{ each config in parameters.MatrixConfigs }}: + - ${{ config }} + - ${{ each config in parameters.AdditionalMatrixConfigs }}: + - ${{ config }} + MatrixFilters: + - TestType=node|browser + - DependencyVersion=^$ + - ${{ each filter in parameters.MatrixFilters }}: + - ${{ filter}} + MatrixReplace: ${{ parameters.MatrixReplace }} + IncludeRelease: ${{ parameters.IncludeRelease }} + variables: + - template: /eng/pipelines/templates/variables/globals.yml@self + - template: /eng/pipelines/templates/variables/image.yml@self - # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch. - - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'), eq(parameters.IncludeRelease,true))}}: - - template: archetype-js-release.yml - parameters: - DependsOn: Build - ServiceDirectory: ${{ parameters.ServiceDirectory }} - TestProxy: ${{ parameters.TestProxy }} - Artifacts: ${{ parameters.Artifacts }} - ${{ if eq(parameters.ServiceDirectory, 'template') }}: - TestPipeline: true - ArtifactName: packages - TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} - TargetDocRepoName: ${{ parameters.TargetDocRepoName }} + - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'), eq(parameters.IncludeRelease,true))}}: + - template: archetype-js-release.yml@self + parameters: + DependsOn: Build + ServiceDirectory: ${{ parameters.ServiceDirectory }} + TestProxy: ${{ parameters.TestProxy }} + Artifacts: ${{ parameters.Artifacts }} + ${{ if eq(parameters.ServiceDirectory, 'template') }}: + TestPipeline: true + ArtifactName: packages + TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} + TargetDocRepoName: ${{ parameters.TargetDocRepoName }} diff --git a/eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml b/eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml new file mode 100644 index 000000000000..fc6777779989 --- /dev/null +++ b/eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml @@ -0,0 +1,122 @@ +parameters: + - name: PackageName + type: string + default: "" + - name: ServiceDirectory + type: string + default: "" + - name: TestResourceDirectories + type: object + default: + - name: EnvVars + type: object + default: {} + - name: MaxParallel + type: number + default: 0 + - name: TimeoutInMinutes + type: number + default: 60 + - name: PublishCodeCoverage + type: boolean + default: false + - name: Location + type: string + default: "" + - name: Clouds + type: string + default: 'Public' + - name: SupportedClouds + type: string + default: 'Public' + - name: UnsupportedClouds + type: string + default: '' + - name: PreSteps + type: object + default: [] + - name: PostSteps + type: object + default: [] + - name: CloudConfig + type: object + default: + Public: + SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) + Preview: + SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources-preview) + Canary: + SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) + Location: 'centraluseuap' + MatrixFilters: + - OSVmImage=.*Ubuntu.* + - DependencyVersion=^$ + UsGov: + SubscriptionConfiguration: $(sub-config-gov-test-resources) + China: + SubscriptionConfiguration: $(sub-config-cn-test-resources) + - name: MatrixConfigs + type: object + default: + - Name: Js_live_test_base + Path: eng/pipelines/templates/stages/platform-matrix.json + Selection: sparse + GenerateVMJobs: true + - name: AdditionalMatrixConfigs + type: object + default: [] + - name: MatrixFilters + type: object + default: [] + - name: MatrixReplace + type: object + default: [] + +stages: + - ${{ each cloud in parameters.CloudConfig }}: + - ${{ if or(contains(parameters.Clouds, cloud.key), and(contains(variables['Build.DefinitionName'], 'tests-weekly'), contains(parameters.SupportedClouds, cloud.key))) }}: + - ${{ if not(contains(parameters.UnsupportedClouds, cloud.key)) }}: + - stage: ${{ cloud.key }} + dependsOn: [] + variables: + - template: /eng/pipelines/templates/variables/image.yml + jobs: + - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml + parameters: + SparseCheckoutPaths: + # JS recording files are implicit excluded here since they are using '.js' file extension. + - "sdk/${{ parameters.ServiceDirectory }}/**/*.json" + JobTemplatePath: /eng/pipelines/templates/jobs/live.tests.yml + OsVmImage: $(LINUXVMIMAGE) + Pool: $(LINUXPOOL) + AdditionalParameters: + PackageName: ${{ parameters.PackageName }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + EnvVars: ${{ parameters.EnvVars }} + MaxParallel: ${{ parameters.MaxParallel }} + TimeoutInMinutes: ${{ parameters.TimeoutInMinutes }} + TestResourceDirectories: ${{ parameters.TestResourceDirectories }} + PublishCodeCoverage: ${{ parameters.PublishCodeCoverage }} + PreSteps: + - ${{ parameters.PreSteps }} + PostSteps: + - ${{ parameters.PostSteps }} + MatrixConfigs: + # Enumerate platforms and additional platforms based on supported clouds (sparse platform<-->cloud matrix). + - ${{ each config in parameters.MatrixConfigs }}: + - ${{ config }} + - ${{ each config in parameters.AdditionalMatrixConfigs }}: + - ${{ config }} + MatrixFilters: + - ${{ each cloudFilter in cloud.value.MatrixFilters }}: + - ${{ cloudFilter }} + - ${{ parameters.MatrixFilters }} + MatrixReplace: + - ${{ each cloudReplace in cloud.value.MatrixReplace }}: + - ${{ cloudReplace }} + - ${{ parameters.MatrixReplace }} + CloudConfig: + SubscriptionConfiguration: ${{ cloud.value.SubscriptionConfiguration }} + SubscriptionConfigurations: ${{ cloud.value.SubscriptionConfigurations }} + Location: ${{ coalesce(parameters.Location, cloud.value.Location) }} + Cloud: ${{ cloud.key }} diff --git a/eng/pipelines/templates/stages/archetype-sdk-tests.yml b/eng/pipelines/templates/stages/archetype-sdk-tests.yml index 9b6828ddb1d3..d2955f2aecd7 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-tests.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-tests.yml @@ -1,10 +1,16 @@ +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release parameters: - name: PackageName type: string - default: "" + default: '' - name: ServiceDirectory type: string - default: "" + default: '' - name: TestResourceDirectories type: object default: @@ -22,13 +28,13 @@ parameters: default: false - name: Location type: string - default: "" + default: '' - name: Clouds type: string - default: 'Public' + default: Public - name: SupportedClouds type: string - default: 'Public' + default: Public - name: UnsupportedClouds type: string default: '' @@ -47,7 +53,7 @@ parameters: SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources-preview) Canary: SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) - Location: 'centraluseuap' + Location: centraluseuap MatrixFilters: - OSVmImage=.*Ubuntu.* - DependencyVersion=^$ @@ -72,47 +78,57 @@ parameters: type: object default: [] -stages: -- ${{ each cloud in parameters.CloudConfig }}: - - ${{ if or(contains(parameters.Clouds, cloud.key), and(contains(variables['Build.DefinitionName'], 'tests-weekly'), contains(parameters.SupportedClouds, cloud.key))) }}: - - ${{ if not(contains(parameters.UnsupportedClouds, cloud.key)) }}: - - stage: ${{ cloud.key }} - dependsOn: [] - jobs: - - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml - parameters: - SparseCheckoutPaths: - # JS recording files are implicit excluded here since they are using '.js' file extension. - - "sdk/${{ parameters.ServiceDirectory }}/**/*.json" - JobTemplatePath: /eng/pipelines/templates/jobs/live.tests.yml - AdditionalParameters: - PackageName: ${{ parameters.PackageName }} - ServiceDirectory: ${{ parameters.ServiceDirectory }} - EnvVars: ${{ parameters.EnvVars }} - MaxParallel: ${{ parameters.MaxParallel }} - TimeoutInMinutes: ${{ parameters.TimeoutInMinutes }} - TestResourceDirectories: ${{ parameters.TestResourceDirectories }} - PublishCodeCoverage: ${{ parameters.PublishCodeCoverage }} - PreSteps: - - ${{ parameters.PreSteps }} - PostSteps: - - ${{ parameters.PostSteps }} - MatrixConfigs: - # Enumerate platforms and additional platforms based on supported clouds (sparse platform<-->cloud matrix). - - ${{ each config in parameters.MatrixConfigs }}: - - ${{ config }} - - ${{ each config in parameters.AdditionalMatrixConfigs }}: - - ${{ config }} - MatrixFilters: - - ${{ each cloudFilter in cloud.value.MatrixFilters }}: - - ${{ cloudFilter }} - - ${{ parameters.MatrixFilters }} - MatrixReplace: - - ${{ each cloudReplace in cloud.value.MatrixReplace }}: - - ${{ cloudReplace }} - - ${{ parameters.MatrixReplace }} - CloudConfig: - SubscriptionConfiguration: ${{ cloud.value.SubscriptionConfiguration }} - SubscriptionConfigurations: ${{ cloud.value.SubscriptionConfigurations }} - Location: ${{ coalesce(parameters.Location, cloud.value.Location) }} - Cloud: ${{ cloud.key }} +extends: + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + ${{ else }}: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + settings: + skipBuildTagsForGitHubPullRequests: true + sdl: + sourceAnalysisPool: + name: azsdk-pool-mms-win-2022-general + image: azsdk-pool-mms-win-2022-1espt + os: windows + eslint: + enabled: false + justificationForDisabling: 'ESLint injected task has failures because it uses an old version of mkdirp. We should not fail for tools not controlled by the repo. See: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=3499746' + psscriptanalyzer: + compiled: true + break: true + policy: M365 + credscan: + suppressionsFile: $(Build.SourcesDirectory)/eng/CredScanSuppression.json + toolVersion: 2.3.12.23 + stages: + - template: archetype-sdk-tests-isolated.yml@self + parameters: + PackageName: ${{ parameters.PackageName }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + TestResourceDirectories: ${{ parameters.TestResourceDirectories }} + EnvVars: ${{ parameters.EnvVars }} + MaxParallel: ${{ parameters.MaxParallel }} + TimeoutInMinutes: ${{ parameters.TimeoutInMinutes }} + PublishCodeCoverage: ${{ parameters.PublishCodeCoverage }} + Location: ${{ parameters.Location }} + Clouds: ${{ parameters.Clouds }} + SupportedClouds: ${{ parameters.SupportedClouds }} + UnsupportedClouds: ${{ parameters.UnsupportedClouds }} + PreSteps: + - ${{ parameters.PreSteps }} + PostSteps: + - ${{ parameters.PostSteps }} + CloudConfig: ${{ parameters.CloudConfig }} + MatrixConfigs: + - ${{ each config in parameters.MatrixConfigs }}: + - ${{ config }} + AdditionalMatrixConfigs: + - ${{ each config in parameters.AdditionalMatrixConfigs }}: + - ${{ config }} + MatrixFilters: + - ${{ each config in parameters.MatrixFilters }}: + - ${{ config }} + MatrixReplace: + - ${{ each config in parameters.MatrixReplace }}: + - ${{ config }} diff --git a/eng/pipelines/templates/stages/cosmos-sdk-client.yml b/eng/pipelines/templates/stages/cosmos-sdk-client.yml index 35410e878afb..d53c28ed80f2 100644 --- a/eng/pipelines/templates/stages/cosmos-sdk-client.yml +++ b/eng/pipelines/templates/stages/cosmos-sdk-client.yml @@ -1,60 +1,90 @@ +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + parameters: -- name: Artifacts - type: object - default: [] -- name: ServiceDirectory - type: string - default: not-specified -- name: RunUnitTests - type: boolean - default: false -- name: TargetDocRepoOwner - type: string - default: MicrosoftDocs -- name: TargetDocRepoName - type: string - default: azure-docs-sdk-node + - name: Artifacts + type: object + default: [] + - name: ServiceDirectory + type: string + default: not-specified + - name: RunUnitTests + type: boolean + default: false + - name: TargetDocRepoOwner + type: string + default: MicrosoftDocs + - name: TargetDocRepoName + type: string + default: azure-docs-sdk-node -variables: - - template: /eng/pipelines/templates/variables/globals.yml +extends: + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + ${{ else }}: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + settings: + skipBuildTagsForGitHubPullRequests: true + sdl: + sourceAnalysisPool: + name: azsdk-pool-mms-win-2022-general + image: azsdk-pool-mms-win-2022-1espt + os: windows + eslint: + enabled: false + justificationForDisabling: 'ESLint injected task has failures because it uses an old version of mkdirp. We should not fail for tools not controlled by the repo. See: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=3499746' + psscriptanalyzer: + compiled: true + break: true + policy: M365 + credscan: + suppressionsFile: $(Build.SourcesDirectory)/eng/CredScanSuppression.json + toolVersion: 2.3.12.23 + stages: + - stage: Build + jobs: + - template: /eng/pipelines/templates/jobs/ci.yml@self + parameters: + Artifacts: ${{parameters.Artifacts}} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + RunUnitTests: ${{ parameters.RunUnitTests }} + MatrixConfigs: + - Name: Javascript_ci_test_base + Path: eng/pipelines/templates/stages/platform-matrix.json + Selection: sparse + GenerateVMJobs: true + variables: + - template: /eng/pipelines/templates/variables/globals.yml@self + - template: /eng/pipelines/templates/variables/image.yml@self -stages: - - stage: Build - jobs: - - template: /eng/pipelines/templates/jobs/ci.yml + - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml@self parameters: - Artifacts: ${{parameters.Artifacts}} - ServiceDirectory: ${{ parameters.ServiceDirectory }} - RunUnitTests: ${{ parameters.RunUnitTests }} - MatrixConfigs: - - Name: Javascript_ci_test_base - Path: eng/pipelines/templates/stages/platform-matrix.json - Selection: sparse - GenerateVMJobs: true - - - template: /eng/pipelines/templates/stages/archetype-sdk-tests.yml - parameters: - PackageName: "@azure/cosmos" - MatrixFilters: - - TestType=node - - DependencyVersion=^$ - - NodeTestVersion=18.x - - Pool=.*mms-win-2022.* - PreSteps: - - template: /eng/pipelines/templates/steps/cosmos-integration-public.yml - PostSteps: - - template: /eng/pipelines/templates/steps/cosmos-additional-steps.yml - EnvVars: - MOCHA_TIMEOUT: 100000 - NODE_TLS_REJECT_UNAUTHORIZED: 0 + PackageName: '@azure/cosmos' + MatrixFilters: + - TestType=node + - DependencyVersion=^$ + - NodeTestVersion=18.x + - Pool=.*mms-win-2022.* + PreSteps: + - template: /eng/pipelines/templates/steps/cosmos-integration-public.yml@self + PostSteps: + - template: /eng/pipelines/templates/steps/cosmos-additional-steps.yml@self + EnvVars: + MOCHA_TIMEOUT: 100000 + NODE_TLS_REJECT_UNAUTHORIZED: 0 - # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch. - - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: - - template: archetype-js-release.yml - parameters: - DependsOn: Build - ServiceDirectory: ${{parameters.ServiceDirectory}} - Artifacts: ${{parameters.Artifacts}} - ArtifactName: packages - TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} - TargetDocRepoName: ${{ parameters.TargetDocRepoName }} + # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch. + - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: + - template: archetype-js-release.yml@self + parameters: + DependsOn: Build + ServiceDirectory: ${{parameters.ServiceDirectory}} + Artifacts: ${{parameters.Artifacts}} + ArtifactName: packages + TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} + TargetDocRepoName: ${{ parameters.TargetDocRepoName }} diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json index 0ffeac1321c7..85b96d2390fd 100644 --- a/eng/pipelines/templates/stages/platform-matrix.json +++ b/eng/pipelines/templates/stages/platform-matrix.json @@ -5,16 +5,16 @@ "matrix": { "Agent": { "windows-2022": { - "OSVmImage": "MMS2022", - "Pool": "azsdk-pool-mms-win-2022-general" + "OSVmImage": "env:WINDOWSVMIMAGE", + "Pool": "env:WINDOWSPOOL" }, "ubuntu-20.04": { - "OSVmImage": "MMSUbuntu20.04", - "Pool": "azsdk-pool-mms-ubuntu-2004-general" + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" }, "macos-11": { - "OSVmImage": "macos-11", - "Pool": "Azure Pipelines" + "OSVmImage": "env:MACVMIMAGE", + "Pool": "env:MACPOOL" } }, "NodeTestVersion": [ @@ -29,8 +29,8 @@ { "Agent": { "windows-2022": { - "OSVmImage": "MMS2022", - "Pool": "azsdk-pool-mms-win-2022-general" + "OSVmImage": "env:WINDOWSVMIMAGE", + "Pool": "env:WINDOWSPOOL" } }, "Scenario": { @@ -53,13 +53,16 @@ { "Agent": { "ubuntu-20.04": { - "OSVmImage": "MMSUbuntu20.04", - "Pool": "azsdk-pool-mms-ubuntu-2004-general" + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" } }, "TestType": "node", "NodeTestVersion": "18.x", - "DependencyVersion": ["max", "min"], + "DependencyVersion": [ + "max", + "min" + ], "TestResultsFiles": "**/test-results.xml" } ] diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index dd3d425d5b3d..f210e844445c 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -20,12 +20,12 @@ steps: ServiceDirectory: "template" TestPipeline: ${{ parameters.TestPipeline }} - - task: PublishPipelineArtifact@1 - condition: succeededOrFailed() - displayName: "Publish Report Artifacts" - inputs: - artifactName: package-diffs - path: $(Build.ArtifactStagingDirectory) + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)' + ArtifactName: 'package-diffs' + SbomEnabled: false + - template: /eng/common/pipelines/templates/steps/verify-readme.yml parameters: @@ -104,9 +104,10 @@ steps: flattenFolders: true displayName: "Copy lint reports" - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: '$(Build.ArtifactStagingDirectory)' ArtifactName: 'reports' + SbomEnabled: false - template: /eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml index a7555dfcd491..d8f28da3ae65 100644 --- a/eng/pipelines/templates/steps/build.yml +++ b/eng/pipelines/templates/steps/build.yml @@ -106,22 +106,11 @@ steps: workingDirectory: $(Pipeline.Workspace) displayName: Create APIView code file - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: '$(Build.ArtifactStagingDirectory)' ArtifactName: 'packages' - - ${{if eq(variables['System.TeamProject'], 'internal') }}: - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Upload Package SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml - parameters: - ArtifactPath: '$(Build.ArtifactStagingDirectory)/_manifest' - ArtifactName: 'manifest' - - template: /eng/pipelines/templates/steps/create-apireview.yml parameters: Artifacts: ${{ parameters.Artifacts }} diff --git a/eng/pipelines/templates/steps/test.yml b/eng/pipelines/templates/steps/test.yml index 9993cd5b9bf8..2aabf185499c 100644 --- a/eng/pipelines/templates/steps/test.yml +++ b/eng/pipelines/templates/steps/test.yml @@ -2,11 +2,12 @@ parameters: Artifacts: [] ServiceDirectory: not-specified TestProxy: true + OSName: '' steps: - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml parameters: - AgentImage: $(OSVmImage) + AgentImage: ${{ parameters.OSName}} - script: | node common/scripts/install-run-rush.js install diff --git a/eng/pipelines/templates/variables/globals.yml b/eng/pipelines/templates/variables/globals.yml index 13a1ea4753ad..5f2759dace7b 100644 --- a/eng/pipelines/templates/variables/globals.yml +++ b/eng/pipelines/templates/variables/globals.yml @@ -5,7 +5,6 @@ variables: skipComponentGovernanceDetection: true coalesceResultFilter: $[ coalesce(variables['packageGlobFilter'], '**') ] ServiceVersion: "" - Package.EnableSBOMSigning: true # Disable CodeQL injections except for where we specifically enable it Codeql.SkipTaskAutoInjection: true # Disable warning until issue 21765 and 21766 are closed diff --git a/eng/pipelines/templates/variables/image.yml b/eng/pipelines/templates/variables/image.yml new file mode 100644 index 000000000000..322e3875f38e --- /dev/null +++ b/eng/pipelines/templates/variables/image.yml @@ -0,0 +1,26 @@ +# Default pool image selection. Set as variable so we can override at pipeline level + +variables: + - name: LINUXPOOL + value: azsdk-pool-mms-ubuntu-2004-general + - name: WINDOWSPOOL + value: azsdk-pool-mms-win-2022-general + - name: MACPOOL + value: Azure Pipelines + + - name: LINUXVMIMAGE + value: azsdk-pool-mms-ubuntu-2004-1espt + - name: LINUXNEXTVMIMAGE + value: azsdk-pool-mms-ubuntu-2204-1espt + - name: WINDOWSVMIMAGE + value: azsdk-pool-mms-win-2022-1espt + - name: MACVMIMAGE + value: macos-11 + + # Values required for pool.os field in 1es pipeline templates + - name: LINUXOS + value: linux + - name: WINDOWSOS + value: windows + - name: MACOS + value: macOS diff --git a/sdk/communication/communication-phone-numbers/phone-numbers-livetest-matrix.json b/sdk/communication/communication-phone-numbers/phone-numbers-livetest-matrix.json index 18c045be47c6..c94e60adb29e 100644 --- a/sdk/communication/communication-phone-numbers/phone-numbers-livetest-matrix.json +++ b/sdk/communication/communication-phone-numbers/phone-numbers-livetest-matrix.json @@ -5,19 +5,19 @@ "matrix": { "Agent": { "windows-2022": { - "OSVmImage": "MMS2022", - "Pool": "azsdk-pool-mms-win-2022-general", + "OSVmImage": "env:WINDOWSVMIMAGE", + "Pool": "env:WINDOWSPOOL", "SKIP_UPDATE_CAPABILITIES_LIVE_TESTS": "true" }, "ubuntu-20.04": { - "OSVmImage": "MMSUbuntu20.04", - "Pool": "azsdk-pool-mms-ubuntu-2004-general", + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL", "AZURE_TEST_AGENT": "UBUNTU_2004_NODE14", "SKIP_UPDATE_CAPABILITIES_LIVE_TESTS": "false" }, "macos-11": { - "OSVmImage": "macos-11", - "Pool": "Azure Pipelines", + "OSVmImage": "env:MACVMIMAGE", + "Pool": "env:MACPOOL", "SKIP_UPDATE_CAPABILITIES_LIVE_TESTS": "true" } }, @@ -29,8 +29,8 @@ { "Agent": { "windows-2022": { - "OSVmImage": "MMS2022", - "Pool": "azsdk-pool-mms-win-2022-general" + "OSVmImage": "env:WINDOWSVMIMAGE", + "Pool": "env:WINDOWSPOOL" } }, "Scenario": { @@ -58,8 +58,8 @@ { "Agent": { "ubuntu-20.04": { - "OSVmImage": "MMSUbuntu20.04", - "Pool": "azsdk-pool-mms-ubuntu-2004-general" + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" } }, "TestType": "node", diff --git a/sdk/keyvault/keyvault-admin/platform-matrix.json b/sdk/keyvault/keyvault-admin/platform-matrix.json index 444880dffe54..edd8ed1e999f 100644 --- a/sdk/keyvault/keyvault-admin/platform-matrix.json +++ b/sdk/keyvault/keyvault-admin/platform-matrix.json @@ -3,8 +3,8 @@ { "Agent": { "ubuntu-20.04_ManagedHSM": { - "OSVmImage": "MMSUbuntu20.04", - "Pool": "azsdk-pool-mms-ubuntu-2004-general", + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL", "ArmTemplateParameters": "@{ enableHsm = $true }" } }, diff --git a/sdk/keyvault/keyvault-certificates/platform-matrix.json b/sdk/keyvault/keyvault-certificates/platform-matrix.json index 6065fc16ea27..f9a69be168d6 100644 --- a/sdk/keyvault/keyvault-certificates/platform-matrix.json +++ b/sdk/keyvault/keyvault-certificates/platform-matrix.json @@ -3,8 +3,8 @@ { "Agent": { "ubuntu-20.04": { - "OSVmImage": "MMSUbuntu20.04", - "Pool": "azsdk-pool-mms-ubuntu-2004-general" + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" } }, "TestType": "node", diff --git a/sdk/keyvault/keyvault-keys/platform-matrix.json b/sdk/keyvault/keyvault-keys/platform-matrix.json index c24868cafd88..25173042d16d 100644 --- a/sdk/keyvault/keyvault-keys/platform-matrix.json +++ b/sdk/keyvault/keyvault-keys/platform-matrix.json @@ -3,8 +3,8 @@ { "Agent": { "ubuntu-20.04_ManagedHSM": { - "OSVmImage": "MMSUbuntu20.04", - "Pool": "azsdk-pool-mms-ubuntu-2004-general", + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL", "ArmTemplateParameters": "@{ enableHsm = $true }" } }, @@ -14,8 +14,8 @@ { "Agent": { "ubuntu-20.04": { - "OSVmImage": "MMSUbuntu20.04", - "Pool": "azsdk-pool-mms-ubuntu-2004-general" + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" } }, "TestType": "node", diff --git a/sdk/keyvault/keyvault-secrets/platform-matrix.json b/sdk/keyvault/keyvault-secrets/platform-matrix.json index 6065fc16ea27..f9a69be168d6 100644 --- a/sdk/keyvault/keyvault-secrets/platform-matrix.json +++ b/sdk/keyvault/keyvault-secrets/platform-matrix.json @@ -3,8 +3,8 @@ { "Agent": { "ubuntu-20.04": { - "OSVmImage": "MMSUbuntu20.04", - "Pool": "azsdk-pool-mms-ubuntu-2004-general" + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" } }, "TestType": "node", diff --git a/sdk/template/template/tests.yml b/sdk/template/template/tests.yml index 030c35cad2d3..4866820c28c0 100644 --- a/sdk/template/template/tests.yml +++ b/sdk/template/template/tests.yml @@ -1,12 +1,13 @@ trigger: none -stages: - - template: /eng/pipelines/templates/stages/archetype-sdk-tests.yml - parameters: - PackageName: "@azure/template" - ServiceDirectory: template - EnvVars: - AZURE_CLIENT_ID: $(TEMPLATE_CLIENT_ID) - AZURE_CLIENT_SECRET: $(TEMPLATE_CLIENT_SECRET) - AZURE_TENANT_ID: $(TEMPLATE_TENANT_ID) - AZURE_SUBSCRIPTION_ID: $(TEMPLATE_SUBSCRIPTION_ID) + +extends: + template: /eng/pipelines/templates/stages/archetype-sdk-tests.yml + parameters: + PackageName: "@azure/template" + ServiceDirectory: template + EnvVars: + AZURE_CLIENT_ID: $(TEMPLATE_CLIENT_ID) + AZURE_CLIENT_SECRET: $(TEMPLATE_CLIENT_SECRET) + AZURE_TENANT_ID: $(TEMPLATE_TENANT_ID) + AZURE_SUBSCRIPTION_ID: $(TEMPLATE_SUBSCRIPTION_ID) From 053a293c6f9f1e8db4c2bd637d983b7d17594ebe Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Mon, 11 Mar 2024 12:05:06 -0700 Subject: [PATCH 2/3] Update eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml Co-authored-by: Ben Broderick Phillips --- eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml b/eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml index fc6777779989..e6eb543b37fa 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml @@ -79,6 +79,7 @@ stages: - stage: ${{ cloud.key }} dependsOn: [] variables: + - template: /eng/pipelines/templates/variables/globals.yml - template: /eng/pipelines/templates/variables/image.yml jobs: - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml From eb34330f6e06476ed916e2ed7b8c2dc03bf36593 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Mon, 11 Mar 2024 12:05:24 -0700 Subject: [PATCH 3/3] Update eng/pipelines/templates/stages/archetype-js-release.yml Co-authored-by: Ben Broderick Phillips --- .../templates/stages/archetype-js-release.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-js-release.yml b/eng/pipelines/templates/stages/archetype-js-release.yml index 50eb0868d942..a357ec660dda 100644 --- a/eng/pipelines/templates/stages/archetype-js-release.yml +++ b/eng/pipelines/templates/stages/archetype-js-release.yml @@ -226,32 +226,24 @@ stages: timeoutInMinutes: 5 - ${{ each artifact in parameters.Artifacts }}: - ${{if ne(artifact.skipPublishDevFeed, 'true')}}: - - pwsh: > + - pwsh: | $detectedPackageName=Get-ChildItem $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.tgz - Write-Host "Detected package name: $($detectedPackageName)" - if ($detectedPackageName -notmatch "-alpha") - { Write-Error "Found non alpha version artifact to publish as dev version. Failing publish step. VersionPolicy should be client or core in rush.json to get alpha version build." exit 1 } - echo "##vso[task.setvariable variable=Package.Archive]$detectedPackageName" - if ('$(Build.Repository.Name)' -eq 'Azure/azure-sdk-for-js') { $npmToken="$(azure-sdk-npm-token)" $registry=${{parameters.Registry}}" } - else { $npmToken="$(azure-sdk-devops-npm-token)" $registry=${{parameters.PrivateRegistry}}" } - echo "##vso[task.setvariable variable=NpmToken]$npmToken" - echo "##vso[task.setvariable variable=Registry]$registry" displayName: Detecting package archive_${{artifact.name}} - task: PowerShell@2