test: Test performance updates

.github/actions/connect-arc/action.yml

@@ -58,4 +58,23 @@ runs:
         -g ${{ env.resource-group }}
         ${{ env.custom-locations-oid }}
         ${{ env.ARC_CLUSTER_CONFIG && format('--kube-config {0}', env.ARC_CLUSTER_CONFIG) || '' }}
+      shell: bash
+    - name: "Wait for cluster connected status"
+      run: |
+        retry_count=0
+        max_retries=5
+        sleep=5
+
+        status=$(az connectedk8s show -n ${{ env.cluster-name }} -g ${{ env.resource-group }} --query "connectivityStatus" -o tsv)
+
+        while [ "$status" != "Connected" ] && [ $retry_count -lt $max_retries ]; do
+          echo "Cluster status is "$status", Retrying in $sleep seconds..."
+          sleep $sleep
+          status=$(az connectedk8s show -n ${{ env.cluster-name }} -g ${{ env.resource-group }} --query "connectivityStatus" -o tsv)
+          retry_count=$((retry_count + 1))
+        done
+        if [ "$status" != "Connected" ]; then
+          echo "Cluster connection failed after $max_retries attempts."
+          exit 1
+        fi
       shell: bash

.github/actions/create-k3s-cluster/action.yml

@@ -0,0 +1,44 @@
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+name: 'Create a k3s cluster in your agent'
+description: |
+  Action to create a k3s cluster in your Github workflow.
+inputs:
+  version:
+    required: false
+    description: 'K3S version string'
+    default: "v1.28.5+k3s1"
+runs:
+  using: 'composite'
+  steps:
+    - name: "Install K3S"
+      env:
+        K3S_VERSION: ${{ inputs.version }}
+      run: |
+        # vars
+        release_url="https://github.com/k3s-io/k3s/releases/download"
+        install_dir="/usr/local/bin/k3s"
+
+        # install k3s
+        curl -Lo $install_dir "$release_url/${{ env.K3S_VERSION }}/k3s"
+        chmod a+x $install_dir
+      shell: bash
+    - name: "Start K3S server"
+      env:
+        # config
+        K3S_KUBECONFIG_MODE: 644
+      run: |
+        # start cluster in background
+        sudo k3s server --cluster-init &
+
+        # allow time for cluster to initialize
+        sleep 15s
+      shell: bash
+    - name: "Write cluster config"
+      run: |
+        mkdir ~/.kube 2> /dev/null
+        sudo k3s kubectl config view --raw > ~/.kube/config
+        chmod 600 ~/.kube/config
+        export KUBECONFIG=~/.kube/config
+      shell: bash

.github/workflows/container_int_test.yml

@@ -0,0 +1,168 @@
+# TODO - consolidate re-used steps between int_test.yml into actions (cluster setup, etc)
+name: E2E Containerized Integration tests
+on:
+  workflow_call:
+    inputs:
+      resource-group:
+        description: Resource group for cluster creation
+        type: string
+        required: true
+        default: ops-cli-int-test-rg
+      custom-locations-oid:
+        description: Object ID of Custom Locations RP
+        type: string
+        required: false
+        default: '51dfe1e8-70c6-4de5-a08e-e18aff23d815'
+    secrets:
+      # required for az login
+      AZURE_CLIENT_ID:
+        required: true
+      AZURE_TENANT_ID:
+        required: true
+      AZURE_SUBSCRIPTION_ID:
+        required: true
+  workflow_dispatch:
+    inputs:
+      resource-group:
+        description: Resource group for cluster creation
+        type: string
+        required: true
+        default: ops-cli-int-test-rg
+      custom-locations-oid:
+        description: Object ID of Custom Locations RP
+        type: string
+        required: false
+        default: '51dfe1e8-70c6-4de5-a08e-e18aff23d815'
+
+permissions:
+  # required for OpenID federation
+  contents: 'read'
+  id-token: 'write'
+
+env:
+  RESOURCE_GROUP: "${{ inputs.resource-group }}"
+  CLUSTER_NAME: "opt${{ github.run_number }}"
+  INSTANCE_NAME: "inst${{ github.run_number }}"
+  STORAGE_NAME: "initstore${{ github.run_number }}"
+  SCHEMAREG_NAME: "init-registry${{ github.run_number }}"
+  CUSTOM_LOCATIONS_OID: ${{ inputs.custom-locations-oid }}
+  K3S_VERSION: "v1.28.5+k3s1"
+
+jobs:
+  test:
+    name: "Run cluster tests"
+    runs-on: ubuntu-22.04
+    steps:
+      - name: "Setup python"
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: "Checkout extension source for build"
+        uses: actions/checkout@v4
+      - name: "Build and install local IoT Ops extension from source"
+        run: |
+          pip install -r dev_requirements.txt
+          python -m setup bdist_wheel -d dist
+          wheel=$(find ./dist/*.whl)
+          az extension add --source $wheel -y
+      - name: "Create k3s cluster"
+        uses: ./.github/actions/create-k3s-cluster
+        with:
+          version: ${{ env.K3S_VERSION }}
+      - name: "Az CLI login"
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+      - name: "ARC connect cluster"
+        uses: azure/azure-iot-ops-cli-extension/.github/actions/connect-arc@dev
+        with:
+          cluster-name: ${{ env.CLUSTER_NAME }}
+          resource-group: ${{ env.RESOURCE_GROUP }}
+          custom-locations-oid: ${{ env.CUSTOM_LOCATIONS_OID }}
+      - name: "Az CLI login refresh"
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+      - name: "Run az iot ops init"
+        run: |
+          # init
+          az iot ops init --cluster ${{ env.CLUSTER_NAME }} -g ${{ env.RESOURCE_GROUP }}
+      - name: "Create storage"
+        id: "create_storage"
+        run: |
+          # create storage account
+          storage_id=$(az storage account create -g ${{ env.RESOURCE_GROUP }} -n ${{ env.STORAGE_NAME }} --enable-hierarchical-namespace --public-network-access Disabled --allow-shared-key-access false --allow-blob-public-access false --default-action Deny --query "id" -o tsv)
+          echo "created storage account with id $storage_id"
+          echo "STORAGE_ID=$storage_id" >> $GITHUB_OUTPUT
+      - name: "Create schema registry"
+        id: "create_schema_registry"
+        run: |
+          # create schema registry
+          schema_registry=$(az iot ops schema registry create -g ${{ env.RESOURCE_GROUP }} -n ${{ env.SCHEMAREG_NAME }} --rn ${{ env.SCHEMAREG_NAME }}-ns --sa-resource-id ${{ steps.create_storage.outputs.STORAGE_ID}} --query "id" -o tsv)
+          echo "created schema registry with id $schema_registry"
+          echo "SCHEMAREGISTRY_ID=$schema_registry" >> $GITHUB_OUTPUT
+      - name: "Run az iot ops create"
+        run: |
+          az iot ops create --cluster ${{ env.CLUSTER_NAME }} -g ${{ env.RESOURCE_GROUP }} -n ${{ env.INSTANCE_NAME }}  --sr-resource-id ${{ steps.create_schema_registry.outputs.SCHEMAREGISTRY_ID }}
+      - name: "Output variables for future steps"
+        id: "env_out"
+        run: |
+          echo "RESOURCE_GROUP=${{ env.RESOURCE_GROUP }}" >> $GITHUB_OUTPUT
+          echo "CLUSTER_NAME=${{ env.CLUSTER_NAME }}" >> $GITHUB_OUTPUT
+      - name: "Containerized tests"
+        env:
+          azext_edge_skip_init: true  # skip init tests in container
+          azext_edge_init_redeployment: false  # ensure no redeployment in container
+          azext_edge_rg: ${{ steps.env_out.outputs.RESOURCE_GROUP }}
+          azext_edge_cluster: ${{ steps.env_out.outputs.CLUSTER_NAME }}
+          azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }}
+        run: |
+          # volume mounts
+          azure_dir=$(realpath ~/.azure)
+          kubeconfig=$(realpath ~/.kube/config)
+          kubeconfig_mount=/root/.kube/config
+          tempLog=$(mktemp -d)
+
+          # env vars
+          envVars=()
+          envVars+=("-e" "azext_edge_skip_init=$azext_edge_skip_init")
+          envVars+=("-e" "azext_edge_init_redeployment=$azext_edge_init_redeployment")
+          envVars+=("-e" "AIO_CLI_INIT_PREFLIGHT_DISABLED=$AIO_CLI_INIT_PREFLIGHT_DISABLED")
+          envVars+=("-e" "azext_edge_rg=$azext_edge_rg")
+          envVars+=("-e" "azext_edge_cluster=$azext_edge_cluster")
+          envVars+=("-e" "azext_edge_instance=$azext_edge_instance")
+          envVars+=("-e" "KUBECONFIG=$kubeconfig_mount")
+
+          # Run tests
+          set +e
+          docker run \
+            --rm \
+            "${envVars[@]}" \
+            -v "$kubeconfig:$kubeconfig_mount:ro" \
+            -v "${azure_dir}:/root/.azure" \
+            -v "${tempLog}:/usr/src/azure-iot-ops/junit" \
+            --network host \
+            $(docker build . -q)
+      - name: "Az CLI login refresh"
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+      - name: "Delete AIO resources"
+        if: ${{ always() }}
+        run: |
+          az iot ops delete -n ${{ env.INSTANCE_NAME }} -g ${{ env.RESOURCE_GROUP }} -y --force
+      - name: "Delete connected cluster"
+        if: ${{ always() }}
+        run: |
+          az resource delete -v --name ${{ env.CLUSTER_NAME }} -g ${{ env.RESOURCE_GROUP }} --resource-type Microsoft.Kubernetes/connectedClusters --verbose
+      - name: "Delete schemaregistry and storage"
+        if: ${{ always() }}
+        run: |
+          az resource delete -v --id ${{ steps.create_storage.outputs.STORAGE_ID }} --verbose --no-wait
+          az resource delete -v --id ${{ steps.create_schema_registry.outputs.SCHEMAREGISTRY_ID }} --verbose --no-wait

.github/workflows/int_test.yml

@@ -40,10 +40,6 @@ on:
         required: true
       AZURE_SUBSCRIPTION_ID:
         required: true
-      # optional --sp-* init params
-      AIO_SP_APP_ID:
-      AIO_SP_OBJECT_ID:
-      AIO_SP_SECRET:
   workflow_dispatch:
     inputs:
       resource-group:
@@ -84,17 +80,14 @@ permissions:
 
 env:
   RESOURCE_GROUP: "${{ inputs.resource-group }}"
+  K3S_VERSION: "v1.28.5+k3s1"
+  CUSTOM_LOCATIONS_OID: "${{ inputs.custom-locations-oid }}"
 
 jobs:
   test:
     env:
       CLUSTER_NAME: "opt${{ github.run_number }}${{ matrix.feature }}"
       INSTANCE_NAME: "inst${{ github.run_number }}${{ matrix.feature }}"
-      CUSTOM_LOCATIONS_OID: ${{ inputs.custom-locations-oid }}
-      EXTENSION_SOURCE_DIRECTORY: "./azure-iot-ops-cli-extension"
-      K3S_VERSION: "v1.28.5+k3s1"
-      CA_FILE: "test-ca.pem"
-      CA_KEY_FILE: "test-ca-key.pem"
     strategy:
       fail-fast: false
       matrix:
@@ -126,8 +119,6 @@ jobs:
             echo "NO_PREFLIGHT=true" >> $GITHUB_OUTPUT
           elif [[ ${{ matrix.feature }} == "syncrules" ]]; then
             echo "CREATE_ARG=--enable-rsync" >> $GITHUB_OUTPUT
-          elif [[ ${{ matrix.feature }} == "redeploy" ]]; then
-            echo "REDEPLOY=True" >> $GITHUB_OUTPUT
           elif [[ ${{ matrix.feature }} == "custom-input" ]]; then
             echo "CREATE_ARG=${{ inputs.runtime-create-args }}" >> $GITHUB_OUTPUT
             echo "INIT_ARG=${{ inputs.runtime-init-args }}" >> $GITHUB_OUTPUT
@@ -147,41 +138,14 @@ jobs:
           python-version: "3.12"
       - name: "Checkout extension source for build"
         uses: actions/checkout@v4
-        with:
-          # ensure source checkout uses our repo instead of calling workflow
-          repository: azure/azure-iot-ops-cli-extension
-          path: ${{ env.EXTENSION_SOURCE_DIRECTORY }}
       - name: "Build and install local IoT Ops extension from source"
         run: |
-          cd ${{ env.EXTENSION_SOURCE_DIRECTORY }}
           pip install -r dev_requirements.txt
           python -m setup bdist_wheel -d dist
           wheel=$(find ./dist/*.whl)
           az extension add --source $wheel -y
       - name: "Create k3s cluster"
-        run: |
-          # vars
-          release_url="https://github.com/k3s-io/k3s/releases/download"
-          install_dir="/usr/local/bin/k3s"
-
-          # install k3s
-          curl -Lo $install_dir "$release_url/${{ env.K3S_VERSION }}/k3s"
-          chmod a+x $install_dir
-
-          # config
-          K3S_KUBECONFIG_MODE="644"
-
-          # start cluster in background
-          sudo k3s server --cluster-init &
-
-          # allow time for cluster to initialize
-          sleep 15s
-      - name: "Write cluster config"
-        run: |
-          mkdir ~/.kube 2> /dev/null
-          sudo k3s kubectl config view --raw > ~/.kube/config
-          chmod 600 ~/.kube/config
-          export KUBECONFIG=~/.kube/config
+        uses: ./.github/actions/create-k3s-cluster
       - name: "Apply custom cluster issuer"
         if: ${{ matrix.feature == 'trustbundle' || contains(inputs.runtime-init-args, '--user-trust') }}
         run: |
@@ -266,16 +230,6 @@ jobs:
           client-id: ${{ secrets.AZURE_CLIENT_ID }}
           tenant-id: ${{ secrets.AZURE_TENANT_ID }}
           subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-      - name: "OIDC Token exchange service"
-        run: |
-          while true; do
-            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
-            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
-            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
-            az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -t ${{ secrets.AZURE_TENANT_ID }} --federated-token $token --output none
-            # Sleep for 4 minutes
-            sleep 240
-          done &
       - name: "ARC connect cluster"
         uses: azure/azure-iot-ops-cli-extension/.github/actions/connect-arc@dev
         with:
@@ -284,14 +238,12 @@ jobs:
           custom-locations-oid: ${{ env.CUSTOM_LOCATIONS_OID }}
       - name: "Tox test environment setup for init"
         run: |
-          cd ${{ env.EXTENSION_SOURCE_DIRECTORY }}
           python -m pip install tox
           tox r -vv -e python-init-int --notest
       - name: "Tox test environment setup for integration tests"
         if: ${{ matrix.feature == 'default' && !inputs.use-container }}
         run: |
-          cd ${{ env.EXTENSION_SOURCE_DIRECTORY }}
-          tox r -vv -e python-int --notest
+          tox r -vv -e "python-all-int" --notest
       - name: "Tox INIT Integration Tests"
         env:
           AIO_CLI_INIT_PREFLIGHT_DISABLED: ${{ steps.init.outputs.NO_PREFLIGHT }}
@@ -301,9 +253,9 @@ jobs:
           azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }}
           azext_edge_init_args: ${{ steps.init.outputs.INIT_ARG }}
           azext_edge_create_args: ${{ steps.init.outputs.CREATE_ARG }}
-          azext_edge_init_redeployment: ${{ steps.init.outputs.REDEPLOY }}
+          # redeploy tested later in this workflow
+          azext_edge_init_redeployment: false
         run: |
-          cd ${{ env.EXTENSION_SOURCE_DIRECTORY }}
           tox r -e python-init-int --skip-pkg-install -- --durations=0
       - name: "Tox Integration Tests"
         if: ${{ matrix.feature == 'default' && !inputs.use-container }}
@@ -312,8 +264,7 @@ jobs:
           azext_edge_cluster: ${{ steps.env_out.outputs.CLUSTER_NAME }}
           azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }}
         run: |
-          cd ${{ env.EXTENSION_SOURCE_DIRECTORY }}
-          tox r -e python-int --skip-pkg-install -- --durations=0
+          tox r -e "python-all-int" --skip-pkg-install -- --durations=0 --dist=loadfile -n auto
           coverage=$(jq .totals.percent_covered coverage.json | cut -c1-4)
           echo "Code coverage: $coverage%" >> $GITHUB_STEP_SUMMARY
       - name: "Containerized tests"
@@ -351,7 +302,7 @@ jobs:
             -v "${azure_dir}:/root/.azure" \
             -v "${tempLog}:/usr/src/azure-iot-ops/junit" \
             --network host \
-            $(docker build ${{ env.EXTENSION_SOURCE_DIRECTORY }} -q)
+            $(docker build . -q)
       - name: "Run smoke tests"
         run: |
           az iot ops support create-bundle
@@ -376,7 +327,6 @@ jobs:
           azext_edge_init_args: ${{ steps.init.outputs.INIT_ARG }}
           azext_edge_create_args: ${{ steps.init.outputs.CREATE_ARG }}
         run: |
-          cd ${{ env.EXTENSION_SOURCE_DIRECTORY }}
           tox r -e python-init-int --skip-pkg-install -- --durations=0
       - name: "Delete AIO resources"
         if: ${{ always() }}