From 03a082bec3ee3745568165970de4dd8d00e95868 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Fri, 27 May 2022 12:28:52 +0800 Subject: [PATCH] [ci] Change artifact reference pipeline to common lib pipeline. (#2294) * [ci] Change artifact reference pipeline to common lib pipeline. --- .../build-docker-sonic-vs-template.yml | 24 ++++--- .azure-pipelines/build-template.yml | 64 ++++++++++++------- .../test-docker-sonic-vs-template.yml | 19 +++--- azure-pipelines.yml | 3 + 4 files changed, 70 insertions(+), 40 deletions(-) diff --git a/.azure-pipelines/build-docker-sonic-vs-template.yml b/.azure-pipelines/build-docker-sonic-vs-template.yml index f47b95c961..ff0ff6c0cb 100644 --- a/.azure-pipelines/build-docker-sonic-vs-template.yml +++ b/.azure-pipelines/build-docker-sonic-vs-template.yml @@ -36,40 +36,47 @@ jobs: inputs: source: specific project: build - pipeline: 9 + pipeline: Azure.sonic-swss-common artifact: ${{ parameters.swss_common_artifact_name }} runVersion: 'latestFromBranch' runBranch: 'refs/heads/$(BUILD_BRANCH)' + path: $(Build.ArtifactStagingDirectory)/download displayName: "Download sonic swss common deb packages" - task: DownloadPipelineArtifact@2 inputs: source: specific project: build - pipeline: 12 + pipeline: Azure.sonic-sairedis artifact: ${{ parameters.sairedis_artifact_name }} runVersion: 'latestFromBranch' runBranch: 'refs/heads/$(BUILD_BRANCH)' + path: $(Build.ArtifactStagingDirectory)/download displayName: "Download sonic sairedis deb packages" - task: DownloadPipelineArtifact@2 inputs: artifact: ${{ parameters.swss_artifact_name }} - displayName: "Download sonic swss artifact" + path: $(Build.ArtifactStagingDirectory)/download + displayName: "Download pre-stage built ${{ parameters.swss_artifact_name }}" - task: DownloadPipelineArtifact@2 inputs: source: specific project: build - pipeline: 142 + pipeline: Azure.sonic-buildimage.official.vs artifact: sonic-buildimage.vs runVersion: 'latestFromBranch' runBranch: 'refs/heads/$(BUILD_BRANCH)' - displayName: "Download sonic buildimage" + path: $(Build.ArtifactStagingDirectory)/download + patterns: '**/target/docker-sonic-vs.gz' + displayName: "Download sonic-buildimage docker-sonic-vs" - script: | + set -ex echo $(Build.DefinitionName).$(Build.BuildNumber) - docker load < ../target/docker-sonic-vs.gz + docker load < $(Build.ArtifactStagingDirectory)/download/target/docker-sonic-vs.gz + mkdir -p .azure-pipelines/docker-sonic-vs/debs - cp -v ../*.deb .azure-pipelines/docker-sonic-vs/debs + cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs pushd .azure-pipelines @@ -78,7 +85,8 @@ jobs: popd docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz - + rm -rf $(Build.ArtifactStagingDirectory)/download + displayName: "Build docker-sonic-vs" - publish: $(Build.ArtifactStagingDirectory)/ artifact: ${{ parameters.artifact_name }} displayName: "Archive sonic docker vs image" diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index d30c510796..f2e577dc2f 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -36,6 +36,9 @@ parameters: type: boolean default: false +- name: common_lib_artifact_name + type: string + jobs: - job: displayName: ${{ parameters.arch }} @@ -75,50 +78,63 @@ jobs: inputs: source: specific project: build - pipeline: 9 + pipeline: Azure.sonic-swss-common artifact: ${{ parameters.swss_common_artifact_name }} runVersion: 'latestFromBranch' runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: '$(Build.SourcesDirectory)/${{ parameters.swss_common_artifact_name }}' + path: $(Build.ArtifactStagingDirectory)/download + patterns: | + libswsscommon_1.0.0_${{ parameters.arch }}.deb + libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb displayName: "Download sonic swss common deb packages" - task: DownloadPipelineArtifact@2 inputs: source: specific project: build - pipeline: 12 + pipeline: Azure.sonic-sairedis artifact: ${{ parameters.sairedis_artifact_name }} runVersion: 'latestFromBranch' runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: '$(Build.SourcesDirectory)/${{ parameters.sairedis_artifact_name }}' + path: $(Build.ArtifactStagingDirectory)/download + patterns: | + libsaivs_*.deb + libsaivs-dev_*.deb + libsairedis_*.deb + libsairedis-dev_*.deb + libsaimetadata_*.deb + libsaimetadata-dev_*.deb + syncd-vs_*.deb displayName: "Download sonic sairedis deb packages" - task: DownloadPipelineArtifact@2 - ${{ if eq(parameters.buildimage_pipeline, 141) }}: - continueOnError: True inputs: source: specific project: build - pipeline: ${{ parameters.buildimage_pipeline }} - artifact: ${{ parameters.buildimage_artifact_name }} + pipeline: Azure.sonic-buildimage.common_libs + artifact: ${{ parameters.common_lib_artifact_name }} runVersion: 'latestFromBranch' runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: '$(Build.SourcesDirectory)/${{ parameters.buildimage_artifact_name }}' - displayName: "Download sonic buildimage deb packages" + path: $(Build.ArtifactStagingDirectory)/download + patterns: | + target/debs/buster/libnl-3-200_*.deb + target/debs/buster/libnl-3-dev_*.deb + target/debs/buster/libnl-genl-3-200_*.deb + target/debs/buster/libnl-genl-3-dev_*.deb + target/debs/buster/libnl-route-3-200_*.deb + target/debs/buster/libnl-route-3-dev_*.deb + target/debs/buster/libnl-nf-3-200_*.deb + target/debs/buster/libnl-nf-3-dev_*.deb + displayName: "Download common libs" - script: | - sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb - sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb - sudo dpkg -i libsaivs_*.deb - sudo dpkg -i libsaivs-dev_*.deb - sudo dpkg -i libsairedis_*.deb - sudo dpkg -i libsairedis-dev_*.deb - sudo dpkg -i libsaimetadata_*.deb - sudo dpkg -i libsaimetadata-dev_*.deb - sudo dpkg -i syncd-vs_*.deb - workingDirectory: $(Pipeline.Workspace) - displayName: "Install sonic swss common and sairedis" - - checkout: self - submodules: true + set -ex + cd download + sudo dpkg -i $(find target/debs/buster -type f) + sudo dpkg -i $(ls *.deb) + cd .. + rm -rf download + workingDirectory: $(Build.ArtifactStagingDirectory) + displayName: "Install libnl3, sonic swss common and sairedis" - script: | - set -x + set -ex tar czf pytest.tgz tests cp -r pytest.tgz $(Build.ArtifactStagingDirectory)/ ./autogen.sh diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml index 95999e171a..3a6979d2b2 100644 --- a/.azure-pipelines/test-docker-sonic-vs-template.yml +++ b/.azure-pipelines/test-docker-sonic-vs-template.yml @@ -17,25 +17,26 @@ jobs: - task: DownloadPipelineArtifact@2 inputs: artifact: docker-sonic-vs - displayName: "Download docker sonic vs image" - + path: $(Build.ArtifactStagingDirectory)/download + displayName: "Download pre-stage built docker-sonic-vs" - task: DownloadPipelineArtifact@2 inputs: source: specific project: build - pipeline: 9 + pipeline: Azure.sonic-swss-common artifact: sonic-swss-common.amd64.ubuntu20_04 runVersion: 'latestFromBranch' runBranch: 'refs/heads/$(BUILD_BRANCH)' + path: $(Build.ArtifactStagingDirectory)/download displayName: "Download sonic swss common deb packages" - script: | - set -x + set -ex sudo .azure-pipelines/build_and_install_module.sh sudo apt-get install -y libhiredis0.14 - sudo dpkg -i --force-confask,confnew ../libswsscommon_1.0.0_amd64.deb || apt-get install -f - sudo dpkg -i ../python3-swsscommon_1.0.0_amd64.deb + sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f + sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb # install packages for vs test sudo apt-get install -y net-tools bridge-utils vlan @@ -44,8 +45,8 @@ jobs: displayName: "Install dependencies" - script: | - set -x - sudo docker load -i ../docker-sonic-vs.gz + set -ex + sudo docker load -i $(Build.ArtifactStagingDirectory)/download/docker-sonic-vs.gz docker ps ip netns list uname -a @@ -53,6 +54,8 @@ jobs: sudo /sbin/ip link del Vrf1 type vrf table 1001 pushd tests sudo py.test -v --force-flaky --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) + + rm -rf $(Build.ArtifactStagingDirectory)/download displayName: "Run vs tests" - task: PublishTestResults@2 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f7c579303d..aa3b6bd94f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -42,6 +42,7 @@ stages: parameters: arch: amd64 sonic_slave: sonic-slave-buster + common_lib_artifact_name: common-lib swss_common_artifact_name: sonic-swss-common sairedis_artifact_name: sonic-sairedis artifact_name: sonic-swss @@ -57,6 +58,7 @@ stages: timeout: 240 pool: sonicbld-armhf sonic_slave: sonic-slave-buster-armhf + common_lib_artifact_name: common-lib.armhf swss_common_artifact_name: sonic-swss-common.armhf sairedis_artifact_name: sonic-sairedis.armhf artifact_name: sonic-swss.armhf @@ -67,6 +69,7 @@ stages: timeout: 240 pool: sonicbld-arm64 sonic_slave: sonic-slave-buster-arm64 + common_lib_artifact_name: common-lib.arm64 swss_common_artifact_name: sonic-swss-common.arm64 sairedis_artifact_name: sonic-sairedis.arm64 artifact_name: sonic-swss.arm64