build: update to 1ES PT

.github/workflows/ci_e2e_workflow.yml

@@ -64,7 +64,7 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
           python -m pip install -U -e .[dev]
- 
+
           if [[ "${{ matrix.python-version }}" != "3.7" ]]; then
              python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[test-http-v2]
           fi
@@ -79,7 +79,7 @@ jobs:
           mkdir logs
       - name: Grant execute permission
         run: |
-            chmod +x .github/Scripts/${{ matrix.test-type }}.sh
+          chmod +x .github/Scripts/${{ matrix.test-type }}.sh
       - name: Running 3.7 ${{ matrix.test-type }}
         if: matrix.python-version == 3.7
         env:

eng/ci/code-mirror.yml

@@ -0,0 +1,18 @@
+trigger:
+  branches:
+    include:
+      - dev
+      - release/*
+
+resources:
+  repositories:
+    - repository: eng
+      type: git
+      name: engineering
+      ref: refs/tags/release
+
+variables:
+  - template: ci/variables/cfs.yml@eng
+
+extends:
+  template: ci/code-mirror.yml@eng

eng/ci/official-build.yml

@@ -0,0 +1,44 @@
+trigger:
+  batch: true
+  branches:
+    include:
+      - dev
+      - release/*
+
+# CI only, does not trigger on PRs.
+pr: none
+
+resources:
+  repositories:
+    - repository: 1es
+      type: git
+      name: 1ESPipelineTemplates/1ESPipelineTemplates
+      ref: refs/tags/release
+    - repository: eng
+      type: git
+      name: engineering
+      ref: refs/tags/release
+
+variables:
+  - template: ci/variables/build.yml@eng
+  - template: ci/variables/cfs.yml@eng
+
+extends:
+  template: v1/1ES.Official.PipelineTemplate.yml@1es
+  parameters:
+    pool:
+      name: 1es-pool-azfunc
+      image: 1es-windows-2022
+      os: windows
+
+    stages:
+      - stage: Build
+
+        jobs:
+          - template: /eng/ci/templates/official/jobs/build-artifacts.yml@self
+
+      - stage: RunTests
+        dependsOn: Build
+
+        jobs:
+          - template: /eng/ci/templates/official/jobs/ci-e2e-tests.yml@self

eng/ci/public-build.yml

@@ -0,0 +1,25 @@
+trigger:
+  batch: true
+  branches:
+    include:
+      - dev
+
+pr:
+  branches:
+    include:
+      - dev
+
+resources:
+  repositories:
+    - repository: 1es
+      type: git
+      name: 1ESPipelineTemplates/1ESPipelineTemplates
+      ref: refs/tags/release
+
+extends:
+  template: v1/1ES.Unofficial.PipelineTemplate.yml@1es
+  parameters:
+    pool:
+      name: 1es-pool-azfunc-public
+      image: 1es-windows-2022
+      os: windows

eng/templates/official/jobs/build-artifacts.yml

@@ -0,0 +1,80 @@
+jobs:
+  - job: "Build"
+    displayName: "Build python worker"
+
+    templateContext:
+      outputParentDirectory: $(Build.ArtifactStagingDirectory)
+      outputs:
+        - output: pipelineArtifact
+          targetPath: $(Build.ArtifactStagingDirectory)
+          artifactName: "drop"
+        - output: nuget
+          condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), eq(variables['UPLOADPACKAGETOPRERELEASEFEED'], true))
+          useDotNetTask: false
+          packagesToPush: "$(Build.ArtifactStagingDirectory)/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/**/*.symbols.nupkg"
+          packageParentPath: "$(Build.ArtifactStagingDirectory)"
+          publishVstsFeed: "e6a70c92-4128-439f-8012-382fe78d6396/f37f760c-aebd-443e-9714-ce725cd427df"
+          nuGetFeedType: "internal"
+          allowPackageConflicts: true
+
+    pool:
+      name: 1es-pool-azfunc
+      image: 1es-windows-2022
+      os: windows
+
+    variables:
+      ${{ if contains(variables['Build.SourceBranch'], '/tags/' ) }}:
+        isTagTemp: true
+      isTag: $[variables.isTagTemp]
+
+    strategy:
+      matrix:
+        Python37V4:
+          pythonVersion: "3.7"
+          workerPath: $(PROD_V4_WORKER_PY)
+        Python38V4:
+          pythonVersion: "3.8"
+          workerPath: $(PROD_V4_WORKER_PY)
+        Python39V4:
+          pythonVersion: "3.9"
+          workerPath: $(PROD_V4_WORKER_PY)
+        Python310V4:
+          pythonVersion: "3.10"
+          workerPath: $(PROD_V4_WORKER_PY)
+        Python311V4:
+          pythonVersion: "3.11"
+          workerPath: $(PROD_V4_WORKER_PY)
+
+    steps:
+      - template: pack/templates/win_env_gen.yml
+        displayName: "Build Windows x64"
+        parameters:
+          pythonVersion: "$(pythonVersion)"
+          workerPath: "$(workerPath)"
+          architecture: "x64"
+          artifactName: "$(pythonVersion)_WINDOWS_X64"
+      - template: pack/templates/win_env_gen.yml
+        displayName: "Build Windows x86"
+        parameters:
+          pythonVersion: "$(pythonVersion)"
+          workerPath: "$(workerPath)"
+          architecture: "x86"
+          artifactName: "$(pythonVersion)_WINDOWS_x86"
+      - template: pack/templates/nix_env_gen.yml
+        displayName: "Build Linux x64"
+        parameters:
+          pythonVersion: "$(pythonVersion)"
+          workerPath: "$(workerPath)"
+          artifactName: "$(pythonVersion)_LINUX_X64"
+      - template: pack/templates/nix_env_gen.yml
+        displayName: "Build OSX X64"
+        parameters:
+          pythonVersion: "$(pythonVersion)"
+          workerPath: "$(workerPath)"
+          artifactName: "$(pythonVersion)_OSX_X64"
+      - template: pack/templates/macos_64_env_gen.yml
+        displayName: "Build OSX ARM64"
+        parameters:
+          pythonVersion: "$(pythonVersion)"
+          workerPath: "$(workerPath)"
+          artifactName: "$(pythonVersion)_OSX_ARM64"

eng/templates/official/jobs/ci-e2e-tests.yml

@@ -0,0 +1,161 @@
+# This workflow will install Python dependencies and run end to end tests with single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: CI E2E tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      archive_webhost_logging:
+        description: "For debugging purposes, archive test webhost logs"
+        required: false
+        default: "false"
+  push:
+    branches: [dev, main, release/*]
+  pull_request:
+    branches: [dev, main, release/*]
+  schedule:
+    # Monday to Friday 3 AM CST build
+    # * is a special character in YAML so you have to quote this string
+    - cron: "0 8 * * 1,2,3,4,5"
+
+jobs:
+  build:
+    name: "Python E2E CI Run"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
+        # Currently runs FWPC E2E tests, classic E2E tests, and DefBindings E2E tests.
+        # To run tests from another script, add the script name to this matrix
+        test-type: [fwpc-e2e-tests, e2e-tests, deferred-bindings-e2e-tests]
+    permissions: read-all
+    steps:
+      - name: Checkout code.
+        uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Set up Dotnet 8.0.x
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: "8.0.x"
+      - name: Install dependencies and the worker
+        run: |
+          retry() {
+            local -r -i max_attempts="$1"; shift
+            local -r cmd="$@"
+            local -i attempt_num=1
+            until $cmd
+            do
+              if (( attempt_num == max_attempts ))
+              then
+                echo "Attempt $attempt_num failed and there are no more attempts left!"
+                return 1
+              else
+                echo "Attempt $attempt_num failed! Trying again in $attempt_num seconds..."
+                sleep 1
+              fi
+            done
+          }
+
+          python -m pip install --upgrade pip
+          python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
+          python -m pip install -U -e .[dev]
+
+          if [[ "${{ matrix.python-version }}" != "3.7" ]]; then
+             python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[test-http-v2]
+          fi
+          if [[ "${{ matrix.python-version }}" != "3.7" && "${{ matrix.python-version }}" != "3.8" ]]; then
+              python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[test-deferred-bindings]
+          fi
+
+          # Retry a couple times to avoid certificate issue
+          retry 5 python setup.py build
+          retry 5 python setup.py webhost --branch-name=dev
+          retry 5 python setup.py extension
+          mkdir logs
+      - name: Grant execute permission
+        run: |
+          chmod +x .github/Scripts/${{ matrix.test-type }}.sh
+      - name: Running 3.7 ${{ matrix.test-type }}
+        if: matrix.python-version == 3.7
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString37 }}
+          AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString37 }}
+          AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString37 }}
+          AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString37 }}
+          AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString37 }}
+          AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString37 }}
+          AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString37 }}
+          ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
+        run: |
+          .github/Scripts/${{ matrix.test-type }}.sh
+      - name: Running 3.8 ${{ matrix.test-type }}
+        if: matrix.python-version == 3.8
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString38 }}
+          AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString38 }}
+          AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString38 }}
+          AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString38 }}
+          AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString38 }}
+          AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString38 }}
+          AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString38 }}
+          ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
+        run: |
+          .github/Scripts/${{ matrix.test-type }}.sh
+      - name: Running 3.9 ${{ matrix.test-type }}
+        if: matrix.python-version == 3.9
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }}
+          AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString39 }}
+          AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString39 }}
+          AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString39 }}
+          AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString39 }}
+          AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString39 }}
+          AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString39 }}
+          ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
+        run: |
+          .github/Scripts/${{ matrix.test-type }}.sh
+      - name: Running 3.10 ${{ matrix.test-type }}
+        if: matrix.python-version == 3.10
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }}
+          AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString310 }}
+          AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString310 }}
+          AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString310 }}
+          AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString310 }}
+          AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString310 }}
+          AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString310 }}
+          ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
+        run: |
+          .github/Scripts/${{ matrix.test-type }}.sh
+      - name: Running 3.11 ${{ matrix.test-type }}
+        if: matrix.python-version == 3.11
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString311 }}
+          AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString311 }}
+          AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString311 }}
+          AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString311 }}
+          AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString311 }}
+          AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString311 }}
+          AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString311 }}
+          ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
+        run: |
+          .github/Scripts/${{ matrix.test-type }}.sh
+      - name: Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml # optional
+          flags: unittests # optional
+          name: codecov # optional
+          fail_ci_if_error: false # optional (default = false)
+      - name: Publish Logs to Artifact
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: Test WebHost Logs ${{ github.run_id }} ${{ matrix.python-version }}
+          path: logs/*.log
+          if-no-files-found: ignore