ci: improve reliability

.github/workflows/release.yml

@@ -17,19 +17,19 @@ jobs:
         config:
           - runtime: osx-x64
             build_args: ""
-            os: ubuntu-latest
+            os: ubuntu-24.04
           - runtime: osx-arm64
             build_args: ""
-            os: ubuntu-latest
+            os: ubuntu-24.04
           - runtime: linux-x64
             build_args: --enable-aot
-            os: ubuntu-latest
+            os: ubuntu-24.04
           - runtime: linux-arm64
             build_args: --enable-aot
-            os: ubuntu-22.04
+            os: ubuntu-24.04-arm
           - runtime: win-x64
             build_args: --enable-aot
-            os: windows-latest
+            os: windows-2022
     name: Build
     runs-on: ${{ matrix.config.os }}
     steps:
@@ -41,19 +41,6 @@ jobs:
         uses: actions/setup-dotnet@v4
         with:
           dotnet-version: '8.0.x'
-      - name: Setup ARM64 Emulator
-        if: matrix.config.runtime == 'linux-arm64'
-        run: |
-          sudo dpkg --add-architecture arm64
-          sudo bash -c 'cat > /etc/apt/sources.list.d/arm64.list <<EOF
-          deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted
-          deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted
-          deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted universe multiverse
-          EOF'
-          sudo sed -i -e 's/deb http/deb [arch=amd64] http/g' /etc/apt/sources.list
-          sudo sed -i -e 's/deb mirror/deb [arch=amd64] mirror/g' /etc/apt/sources.list
-          sudo apt update
-          sudo apt install -y clang llvm binutils-aarch64-linux-gnu gcc-aarch64-linux-gnu zlib1g-dev:arm64
       - name: Build .NET project
         shell: bash
         run: python3 ./scripts/build.py "${GITHUB_REF_NAME}" ${{ matrix.config.runtime }} ${{ matrix.config.build_args }}
@@ -91,7 +78,7 @@ jobs:
           retention-days: 1
   release:
     name: Release
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     needs: macos_codesign
     steps:
       - name: Checkout

.github/workflows/test.yml

@@ -41,9 +41,9 @@ jobs:
           VALIDATE_MARKDOWN: false
           VALIDATE_JSCPD: false
           VALIDATE_SHELL_SHFMT: false
-  test:
-    name: Unit Testing and Build
-    runs-on: ubuntu-latest
+  test-linux:
+    name: Unit Testing and Build on Linux x64
+    runs-on: ubuntu-24.04
     timeout-minutes: 5
     permissions:
       contents: read
@@ -82,11 +82,63 @@ jobs:
           name: darwin-amd64-binary
           path: ./bin/artifacts/notation-azure-kv_0.0.1_darwin_amd64.tar.gz
           retention-days: 1
+  test-linux-arm:
+    name: Unit Testing and Build on Linux arm64
+    runs-on: ubuntu-24.04-arm
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '8.0.x'
+      - name: Check out code into the project directory
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Run unit tests
+        run: make test
+      - name: Build Linux Binary
+        run: |
+          # the binary will be used in E2E test
+          python3 ./scripts/build.py v0.0.1 linux-arm64 --enable-aot
+      - name: Upload Linux artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: linux-arm64-binary
+          path: ./bin/artifacts/notation-azure-kv_0.0.1_linux_arm64.tar.gz
+          retention-days: 1
+  test-windows:
+    name: Unit Testing and Build on windows x64
+    runs-on: windows-2022
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '8.0.x'
+      - name: Check out code into the project directory
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Run unit tests
+        run: make test
+      - name: Build Windows Binary
+        run: python3 ./scripts/build.py v0.0.1 win-x64 --enable-aot
+      - name: Upload Linux artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: windows-amd64-binary
+          path: ./bin/artifacts/notation-azure-kv_0.0.1_windows_amd64.zip
+          retention-days: 1
   e2e-mariner-container:
     name: E2E testing for Mariner container
     runs-on: ubuntu-latest
     environment: E2E
-    needs: test
+    needs: test-linux
     steps:
       - name: Check out code into the project directory
         uses: actions/checkout@v4
@@ -128,24 +180,35 @@ jobs:
           AZURE_FEDERATED_TOKEN_FILE: ./federated_token
   e2e-linux:
     name: E2E testing on Linux
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.config.os }}
+    strategy:
+      matrix:
+        config:
+          - os: ubuntu-24.04
+            arch: amd64
+          - os: ubuntu-22.04
+            arch: amd64
+          - os: ubuntu-24.04-arm
+            arch: arm64
+          - os: ubuntu-22.04-arm
+            arch: arm64
     environment: E2E
-    needs: test
+    needs: [test-linux, test-linux-arm]
     steps:
       - name: Check out code into the project directory
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: actions/download-artifact@v4
         with:
-          name: linux-amd64-binary
+          name: linux-${{ matrix.config.arch }}-binary
           path: ./bin/artifacts
       - name: Run download server locally
         run: |
           nohup python3 -m http.server --directory ./bin/artifacts/ &
 
           # prepare the environment variables for E2E
-          artifactName=notation-azure-kv_0.0.1_linux_amd64.tar.gz
+          artifactName=notation-azure-kv_0.0.1_linux_${{ matrix.config.arch }}.tar.gz
           checksum=$(shasum -a 256 "./bin/artifacts/$artifactName" | awk '{print $1}')
           echo "pluginChecksum=$checksum" >> "$GITHUB_ENV"
           echo "pluginDownloadURL=http://localhost:8000/$artifactName" >> "$GITHUB_ENV"
@@ -168,20 +231,21 @@ jobs:
           pluginChecksum: ${{ env.pluginChecksum }}
   e2e-windows:
     name: E2E testing on Windows
-    runs-on: windows-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-2022, windows-2025]
     environment: E2E
-    needs: test
+    needs: test-windows
     steps:
       - name: Check out code into the project directory
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Setup .NET
-        uses: actions/setup-dotnet@v4
+      - uses: actions/download-artifact@v4
         with:
-          dotnet-version: '8.0.x'
-      - name: Build Windows Binary
-        run: python3 ./scripts/build.py v0.0.1 win-x64 --enable-aot
+          name: windows-amd64-binary
+          path: ./bin/artifacts
       - name: Run download server locally
         run: |
           # wsl bash
@@ -215,7 +279,7 @@ jobs:
     name: E2E testing on macOS
     runs-on: macos-13
     environment: E2E
-    needs: test
+    needs: test-linux
     steps:
       - name: Check out code into the project directory
         uses: actions/checkout@v4

Notation.Plugin.AzureKeyVault.Tests/ProgramTests.cs

@@ -97,10 +97,10 @@ internal interface IResponseMock
 
         [Theory]
         [InlineData(200, "{\"error\":{\"message\":\"TestErrorMessage\"}}", "TestErrorMessage")]
-        [InlineData(500, "{\"error\":{\"message\":\"TestErrorMessage\"}", "Service request failed.\nStatus: 500\n")]
-        [InlineData(500, "{\"error2\":{\"message\":\"TestErrorMessage\"}}", "Service request failed.\nStatus: 500\n")]
-        [InlineData(500, "{\"error\":{\"message2\":\"TestErrorMessage\"}}", "Service request failed.\nStatus: 500\n")]
-        [InlineData(500, "{\"error\":{\"message\":\"\"}}", "\nStatus: 500\n")]
+        [InlineData(500, "{\"error\":{\"message\":\"TestErrorMessage\"}", "Status: 500")]
+        [InlineData(500, "{\"error2\":{\"message\":\"TestErrorMessage\"}}", "Status: 500")]
+        [InlineData(500, "{\"error\":{\"message2\":\"TestErrorMessage\"}}", "Status: 500")]
+        [InlineData(500, "{\"error\":{\"message\":\"\"}}", "Status: 500")]
         public void HandleAzureException(int code, string content, string expectedErrorMessage)
         {
             // Arrange
@@ -124,7 +124,7 @@ public void HandleAzureException(int code, string content, string expectedErrorM
             var errorResponse = Program.HandleAzureException(exception);
 
             // Assert exit code 1
-            Assert.StartsWith(expectedErrorMessage, errorResponse.ErrorMessage);
+            Assert.Contains(expectedErrorMessage, errorResponse.ErrorMessage);
             Assert.Equal("ERROR", errorResponse.ErrorCode);
         }
     }