Support M1 build workflow in release pipeline

.github/workflows/build_m1.yml

@@ -3,13 +3,8 @@
 
 name: Build AzCopy for M1 Mac
 
-on:
+on: 
   workflow_dispatch:
-    inputs:
-      m1_url:
-        description: 'Target object in storage without SAS'
-        required: true
-        type: string
 
 permissions:
   id-token: write
@@ -28,14 +23,12 @@ jobs:
           go-version: '1.22.5'
 
       - name: Build
-        run: CGO_ENABLED=1 go build -o azcopy_darwin_arm64
+        run: |
+          mkdir -p output
+          CGO_ENABLED=1 go build -o output/azcopy_darwin_m1_arm64
 
-      - name: Azure Login
-        uses: Azure/[email protected]
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
         with:
-          client-id: ${{ secrets.AZURE_CLIENT_ID }}
-          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
-          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-
-      - name: Upload to AzCopy to M1 container
-        run: AZCOPY_AUTO_LOGIN_TYPE=AzCLI ./azcopy_darwin_arm64 copy "azcopy_darwin_arm64" "${{ inputs.m1_url }}"
+          name: build-artifacts
+          path: output/

release-pipeline.yml

@@ -12,6 +12,11 @@ parameters:
     type: boolean
     default: false
 
+  - name: publish_to_m1_container
+    displayName: 'Publish To M1 Container'
+    type: boolean
+    default: false      
+
   - name: post_release
     displayName: 'Post Release on Github'
     type: boolean
@@ -532,7 +537,85 @@ stages:
               verbosity: 'Verbose'
               alertWarningLevel: 'High'
 
-      #TODO: support build for mac m1
+      - job: Set_5
+        timeoutInMinutes: 360
+        strategy:
+          matrix:
+            MacOS-14-ARM64:
+              imageName: 'ubuntu-latest'
+              type: 'linux'
+        pool:
+          vmImage: $(imageName)
+
+        variables:
+          - group: AZCOPY_TESTS_VAR
+          - name: root_dir
+            value: '$(System.DefaultWorkingDirectory)'
+          - name: work_dir
+            value: '$(Build.SourcesDirectory)'
+          - name: archives
+            value: $(root_dir)/archives
+          - name: Publish_To_M1_Container
+            value: ${{ parameters.publish_to_m1_container }}
+
+        steps:
+          - task: GoTool@0
+            env:
+              GO111MODULE: 'on'
+            inputs:
+              version: $(AZCOPY_GOLANG_VERSION)
+
+          - script: |
+              echo 'Running GO Vet'
+              go vet 
+            displayName: 'Golang Vet'
+            workingDirectory: $(System.DefaultWorkingDirectory)
+
+          - script: |
+              CGO_ENABLED=1 go build -o "$(root_dir)/azcopy"
+            displayName: 'Generate build'
+
+          - template: setup/azcopy_version_check_linux.yml
+            parameters:
+              azcopy_executable: $(root_dir)/azcopy
+
+          - template: setup/trigger_m1_build.yml
+
+          - script: |
+              mkdir -p $(archives)
+              darwin_arm64_m1_dir="$(work_dir)/azcopy_darwin_m1_arm64_$(azcopy_version)"
+
+              echo "##vso[task.setvariable variable=darwin_arm64_m1_dir]$darwin_arm64_m1_dir"
+              mkdir -p $darwin_arm64_m1_dir
+
+              cp NOTICE.txt $darwin_arm64_m1_dir
+              cp $(Build.ArtifactStagingDirectory)/azcopy  $darwin_arm64_m1_dir
+            displayName: 'Copy required files for packaging'
+
+          - task: ArchiveFiles@2
+            displayName: 'Archive MacOS M1_ARM64 Build (must happen before signing/notarization)'
+            inputs:
+              rootFolderOrFile: '$(darwin_arm64_m1_dir)'
+              archiveFile: '$(archives)/azcopy_darwin_m1_arm64_$(azcopy_version).zip'
+            continueOnError: true
+
+          - script: |
+              rm -rf $(Build.ArtifactStagingDirectory)/azcopy
+              cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory)
+            displayName: 'Copy zip to staging directory'
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              artifactName: 'azCopy-mac-temp'
+            displayName: 'Publish Artifacts'
+            condition: succeededOrFailed()
+
+          - task: ComponentGovernanceComponentDetection@0
+            inputs:
+              scanType: 'Register'
+              verbosity: 'Verbose'
+              alertWarningLevel: 'High'
+
     # BuildArtifacts end here
 
   - stage: SignArtifacts
@@ -867,8 +950,10 @@ stages:
           - script: |
               mkdir -p unzipped_amd64
               mkdir -p unzipped_arm64
+              mkdir -p unzipped_m1_arm64
               unzip -o azcopy*amd64*.zip -d unzipped_amd64
-              unzip -o azcopy*arm64*.zip -d unzipped_arm64
+              unzip -o azcopy_darwin_arm64*.zip -d unzipped_arm64
+              unzip -o azcopy_darwin_m1_arm64*.zip -d unzipped_m1_arm64
 
               # Use find to retrieve the folder name
               amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*")
@@ -877,6 +962,9 @@ stages:
               arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*")
               echo "ARM64 Folder name: $arm64FolderName"
 
+              m1Arm64FolderName=$(find unzipped_m1_arm64 -maxdepth 1 -type d -name "azcopy_darwin_m1_arm64*")
+              echo "ARM64 M1 Folder name: $m1Arm64FolderName"
+
               # Check if there is exactly one matching directory for amd64
               if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then
                   echo "Found matching AMD64 directory: $amd64FolderName"
@@ -899,6 +987,18 @@ stages:
                   exit 1
               fi
 
+              cd $(Build.ArtifactStagingDirectory)
+
+              # Check if there is exactly one matching directory for arm64
+              if [ $(echo "$m1Arm64FolderName" | wc -l) -eq 1 ]; then
+                  echo "Found matching ARM64 M1 directory: $m1Arm64FolderName"
+                  cd "$m1Arm64FolderName"
+                  mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_m1_arm64
+              else
+                  echo "Error: Expected exactly one matching ARM64 M1 directory, found $(echo "$m1Arm64FolderName" | wc -l)."
+                  exit 1
+              fi
+
               rm -rf $(Build.ArtifactStagingDirectory)/unzipped*
               rm -rf $(Build.ArtifactStagingDirectory)/*.zip
               sudo ls -ltR $(Build.ArtifactStagingDirectory)
@@ -1549,6 +1649,8 @@ stages:
             value: 'azcopyvnextrelease'
           - name: Publish_To_Container
             value: ${{ parameters.publish_to_container }}
+          - name: Publish_To_M1_Container
+            value: ${{ parameters.publish_to_m1_container }}
           - name: work_dir
             value: '$(Build.SourcesDirectory)'
 
@@ -1639,6 +1741,11 @@ stages:
                 AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true
                 AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true
 
+                if [ $(Publish_To_M1_Container) = "True" ]; then
+                  m1_container_url="https://azcopyvnextrelease.blob.core.windows.net/azcopy-m1-drops/azcopy-$azcopy_version-$today"
+                  AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" "$m1_container_url" --put-md5=true
+                fi
+
   - ${{ if eq(parameters.post_release, true) }}:
     - stage: ReleaseArtifacts
       dependsOn: TestArtifacts

setup/trigger_m1_build.yml

@@ -0,0 +1,132 @@
+steps:
+  - task: Bash@3
+    inputs:
+      targetType: 'inline'
+      script: |
+        echo "Triggering GitHub Actions workflow"
+
+        GITHUB_OWNER="Azure"
+        GITHUB_REPO="azure-storage-azcopy"
+        GITHUB_WORKFLOW_FILE="build_m1.yml"
+        GITHUB_REF="main"
+        GITHUB_WORKFLOW_NAME="Build AzCopy for M1 Mac"
+        ARTIFACTS_DIR="downloaded_artifacts"  # Directory to save the downloaded artifacts
+
+        GITHUB_API_URL="https://api.github.com/repos/$GITHUB_OWNER/$GITHUB_REPO/actions/workflows/$GITHUB_WORKFLOW_FILE/dispatches"
+
+        # Create the JSON payload
+        PAYLOAD=$(jq -n --arg ref "$GITHUB_REF" '{ref: $ref}')
+
+        # Trigger the workflow
+        response=$(curl -w "%{http_code}" -X POST -H "Authorization: Bearer $(GITHUB_TOKEN)" -H "Accept: application/vnd.github.v3+json" $GITHUB_API_URL -d "$PAYLOAD")
+        echo "response: $response"
+
+        if [ $response -eq 204 ]; then
+          echo "Successfully triggered the workflow"
+        else
+          echo "Failed to trigger the workflow $response"
+          exit 1
+        fi
+
+
+        # GitHub API URL to list workflows
+        WORKFLOWS_URL="https://api.github.com/repos/$GITHUB_OWNER/$GITHUB_REPO/actions/workflows"
+
+        # Retrieve workflows
+        response=$(curl -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer $(GITHUB_TOKEN)" "$WORKFLOWS_URL")
+
+        # Extract the workflow ID for the specified workflow name
+        workflow_id=$(echo $response | jq -r --arg workflow_name "$GITHUB_WORKFLOW_NAME" '.workflows[] | select(.name == $workflow_name) | .id')
+
+        if [ -z "$workflow_id" ]; then
+          echo "Workflow not found"
+          exit 1
+        else
+          echo "Workflow ID: $workflow_id"
+        fi
+
+        # Interval between polls (in seconds)
+        POLL_INTERVAL=15
+
+        # GitHub API URL to list workflow runs
+        WORKFLOW_RUNS_URL="https://api.github.com/repos/$GITHUB_OWNER/$GITHUB_REPO/actions/workflows/$workflow_id/runs"
+
+        # Function to check the status of the latest workflow run
+        check_status() {
+          runs_response=$(curl -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer $(GITHUB_TOKEN)" "$WORKFLOW_RUNS_URL")
+
+          latest_run_status=$(echo $runs_response | jq -r '.workflow_runs[0].status')
+          latest_run_conclusion=$(echo $runs_response | jq -r '.workflow_runs[0].conclusion')
+          latest_run_id=$(echo $runs_response | jq -r '.workflow_runs[0].id')
+
+          if [ -z "$latest_run_id" ]; then
+            echo "No runs found for the workflow"
+            exit 1
+          else
+            echo "Latest Run ID: $latest_run_id"
+            echo "Latest Run Status: $latest_run_status"
+            echo "Latest Run Conclusion: $latest_run_conclusion"
+          fi
+
+          if [ "$latest_run_status" == "completed" ]; then
+            if [ "$latest_run_conclusion" == "success" ]; then
+              echo "Workflow run succeeded."
+              download_artifacts_and_extract $latest_run_id
+              exit 0
+            else
+              echo "Workflow run failed or was cancelled."
+              exit 1
+            fi
+          fi
+        }
+
+        # Function to download the artifacts from the specified workflow run
+        download_artifacts_and_extract() {
+          local run_id=$1
+          ARTIFACTS_URL="https://api.github.com/repos/$GITHUB_OWNER/$GITHUB_REPO/actions/runs/$run_id/artifacts"
+
+          # Create directory for artifacts
+          mkdir -p $ARTIFACTS_DIR
+
+          # Retrieve the list of artifacts
+          artifacts_response=$(curl -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer $(GITHUB_TOKEN)" "$ARTIFACTS_URL")
+
+          # Extract the artifact download URLs
+          artifact_count=$(echo $artifacts_response | jq '.artifacts | length')
+          echo "Found $artifact_count artifacts."
+
+          if [ $artifact_count -eq 1 ]; then
+            artifact_name=$(echo $artifacts_response | jq -r ".artifacts[$i].name")
+            artifact_url=$(echo $artifacts_response | jq -r ".artifacts[$i].archive_download_url")
+
+            echo "Downloading artifact: $artifact_name"
+
+            download_response=$(curl -w "%{http_code}" -L -o "$ARTIFACTS_DIR/$artifact_name.zip" -H "Authorization: Bearer $(GITHUB_TOKEN)" "$artifact_url")
+            echo "download response: $download_response"
+
+            if [ $? -eq 0 ]; then
+              echo "Successfully downloaded $artifact_name to $ARTIFACTS_DIR"
+              # Extract the artifact
+              mkdir -p $ARTIFACTS_DIR/unzipped_artifacts
+              ls -ltR $ARTIFACTS_DIR/
+              unzip -o $ARTIFACTS_DIR/$artifact_name* -d "$ARTIFACTS_DIR/unzipped_artifacts"
+              mv $ARTIFACTS_DIR/unzipped_artifacts/azcopy_darwin_m1_arm64 $(Build.ArtifactStagingDirectory)/azcopy
+            else
+              echo "Failed to download $artifact_name"
+              exit 1
+            fi
+          else
+            echo "No artifacts found for the specified workflow run."
+            exit 1
+          fi
+        }
+
+        # Poll the workflow status until it completes
+        while true; do
+          check_status
+          echo "Workflow run is still in progress. Checking again in $POLL_INTERVAL seconds..."
+          sleep $POLL_INTERVAL
+        done
+
+    displayName: 'Trigger GitHub Actions workflow'
+