Initial commit

iSPIRT · Oct 8, 2023 · 9d86238 · 9d86238
commit 9d86238
Show file tree

Hide file tree

Showing 60 changed files with 15,941 additions and 0 deletions.
diff --git a/.github/workflows/ci-local.yml b/.github/workflows/ci-local.yml
@@ -0,0 +1,46 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+
+name: Deploy COVID scenario in Docker
+
+on: [workflow_dispatch]
+
+env:
+  CONTAINER_REGISTRY: ${{ vars.CONTAINER_REGISTRY }}
+
+jobs:
+  deploy-ccr-covid:
+    runs-on: [self-hosted, linux, X64]
+    steps:
+      - uses: AutoModality/action-clean@v1
+      - uses: actions/checkout@v3
+
+      - name: Update submodules
+        run: git submodule update --init --recursive
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }}
+          password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }}
+
+      - name: Add user to docker group
+        run: sudo usermod -aG docker $USER
+
+      - name: Pull container images
+        run: cd ${{ github.workspace }}/ci && ./pull-containers.sh
+
+      - name: Run pre-processing
+        run: cd ./scenarios/covid/deployment/docker && ./preprocess.sh
+
+      - name: Run model saving
+        run: cd ./scenarios/covid/deployment/docker && ./save-model.sh
+
+      - name: Run training
+        run: cd ./scenarios/covid/deployment/docker && ./train.sh
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,104 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+
+name: Deploy COVID scenario to ACI
+
+on: 
+  workflow_dispatch:
+    inputs: 
+      contract:
+        description: "Contract ID"
+        required: true
+
+env:
+  CONTAINER_REGISTRY: ${{ vars.CONTAINER_REGISTRY }}
+  CONTAINER_REGISTRY_USERNAME: ${{ secrets.CONTAINER_REGISTRY_USERNAME }}
+  CONTAINER_REGISTRY_PASSWORD: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }}
+  AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
+  AZURE_STORAGE_ACCOUNT_NAME: ${{ vars.AZURE_STORAGE_ACCOUNT_NAME }}
+  AZURE_ICMR_CONTAINER_NAME: ${{ vars.AZURE_ICMR_CONTAINER_NAME }}
+  AZURE_COWIN_CONTAINER_NAME: ${{ vars.AZURE_COWIN_CONTAINER_NAME }}
+  AZURE_INDEX_CONTAINER_NAME: ${{ vars.AZURE_INDEX_CONTAINER_NAME }}
+  AZURE_MODEL_CONTAINER_NAME: ${{ vars.AZURE_MODEL_CONTAINER_NAME }}
+  AZURE_OUTPUT_CONTAINER_NAME: ${{ vars.AZURE_OUTPUT_CONTAINER_NAME }}
+  AZURE_KEYVAULT_ENDPOINT: ${{ vars.AZURE_KEYVAULT_ENDPOINT }}
+  TOOLS_HOME: ${{ github.workspace }}/external/confidential-sidecar-containers/tools
+  DATA_DIRECTORY: ${{ github.workspace}}/scenarios/covid/data
+  CONTRACT_SERVICE_URL: ${{ vars.CONTRACT_SERVICE_URL }}
+
+jobs:
+  deploy-ccr-covid-aci:
+    runs-on: [self-hosted, linux, X64]
+    steps:
+      - uses: AutoModality/action-clean@v1
+      - uses: actions/checkout@v3
+
+      - uses: pietrobolcato/install-azure-cli-action@main
+
+      - name: Update submodules
+        run: git submodule update --init --recursive
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.19.x'
+
+      - name: Install jq
+        run: sudo apt install -y jq
+
+      - name: Log in with Azure
+        uses: azure/login@v1
+        with:
+          creds: '${{ secrets.AZURE_CREDENTIALS }}'
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }}
+          password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }}
+
+      - name: Install az confcom extension
+        run:  az extension add --name confcom -y
+
+      - name: Add user to docker group
+        run: sudo usermod -aG docker $USER
+
+      - name: Run pre-processing
+        run: cd ${{ github.workspace }}/scenarios/covid/deployment/docker && ./preprocess.sh
+
+      - name: Run model saving
+        run: cd ${{ github.workspace }}/scenarios/covid/deployment/docker && ./save-model.sh
+
+      - name: Pull container images for generating policy
+        run: cd ${{ github.workspace }}/ci && ./pull-containers.sh
+
+      - name: Import data and model encryption keys with key release policies
+        run: cd ${{ github.workspace }}/scenarios/covid/data && ./3-import-keys.sh
+
+      - name: Encrypt data and models
+        run: cd ${{ github.workspace }}/scenarios/covid/data && ./4-encrypt-data.sh
+
+      - name: Upload data and model
+        run: cd ${{ github.workspace }}/scenarios/covid/data && ./5-upload-encrypted-data.sh
+
+      - name: Run training
+        run: cd ${{ github.workspace }}/scenarios/covid/deployment/aci && ./deploy.sh -c ${{ github.event.inputs.contract }} -q ../../config/query_config.json -m ../../config/model_config.json
+
+      - name: Dump training container logs
+        run: sleep 200 && az container logs --name depa-training-covid --resource-group $AZURE_RESOURCE_GROUP --container-name depa-training
+
+      - name: Dump sidecar container logs
+        run: az container logs --name depa-training-covid --resource-group $AZURE_RESOURCE_GROUP --container-name encrypted-storage-sidecar
+
+      - name: Download and decrypt model
+        run: cd ${{ github.workspace }}/scenarios/covid/data && ./6-download-decrypt-model.sh
+
+      - name: Cleanup data directory
+        run: sudo rm -rf $DATA_DIRECTORY
+        if: ${{ always() }}
diff --git a/.github/workflows/contract-service.yml b/.github/workflows/contract-service.yml
@@ -0,0 +1,49 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+
+name: Deploy Contract Service 
+on: [workflow_dispatch]
+
+env:
+  CONTAINER_REGISTRY: ${{ vars.CONTAINER_REGISTRY }}
+  CONTAINER_REGISTRY_USERNAME: ${{ secrets.CONTAINER_REGISTRY_USERNAME }}
+  CONTAINER_REGISTRY_PASSWORD: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }}
+  PLATFORM: "virtual"
+  CONTAINER_NAME: "contract-ledger"
+
+jobs:    
+  deploy-contract-service:
+    runs-on: [self-hosted, linux, X64]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Update submodules
+        run: git submodule update --init --recursive
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }}
+          password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }}
+
+      - name: Setup python venv
+        run: sudo apt install python3.8-venv -y
+
+      - name: Stop previous instances
+        run: docker stop $CONTAINER_NAME || true 
+
+      - name: Build contract service container
+        run: cd ${{ github.workspace }}/implementation/external/contract-ledger && ./docker/build.sh
+
+      - name: Run contract service
+        run: cd ${{ github.workspace }}/implementation/external/contract-ledger && ./docker/run-dev.sh 
+
+      - name: Wait until service is running
+        run: >
+          timeout 60 bash -c "while ! curl -s -f -k https://localhost:8000/parameters > /dev/null; do echo \"Wating...\"; sleep 10; done"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,99 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+
+name: Publish Images
+
+on: [release]
+
+env:
+  REGISTRY: ${{ vars.CONTAINER_REGISTRY }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - dockerfile: ./scenarios/covid/ci/Dockerfile.index
+            name: preprocess-index
+            context: ./scenarios/covid/src
+            buildargs: |
+          - dockerfile: ./scenarios/covid/ci/Dockerfile.icmr
+            name: preprocess-icmr
+            context: ./scenarios/covid/src
+            buildargs: |
+          - dockerfile: ./scenarios/covid/ci/Dockerfile.cowin
+            name: preprocess-cowin
+            context: ./scenarios/covid/src
+            buildargs: |
+          - dockerfile: ./scenarios/covid/ci/Dockerfile.modelsave
+            name: ccr-model-save
+            context: ./scenarios/covid/src
+            buildargs: |
+          - dockerfile: ./ci/Dockerfile.encfs
+            name: depa-training-encfs
+            context: ./
+            buildargs: |
+          - dockerfile: ./ci/Dockerfile.train
+            name: depa-training
+            context: ./src
+            buildargs: |
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }}
+          password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ matrix.name }}
+
+      - name: Build encrypted filesystem artifacts
+        run: cd ${{ github.workspace }}/external/confidential-sidecar-containers && ./buildall.sh
+
+      - name: Build contract ledger client library
+        run: cd ${{ github.workspace }}//external/contract-ledger/pyscitt && python3 setup.py bdist_wheel
+
+      - name: Build and push container image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: ${{ matrix.context }}
+          file: ${{ matrix.dockerfile }}
+          build-args: ${{ matrix.buildargs }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Set SCITT_VERSION_OVERRIDE
+        run: echo "SCITT_VERSION_OVERRIDE=$(git describe --tags --match="*.*.*")" >> "$GITHUB_ENV"
+
+      - name: Build contract service container
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          file: ${{ github.workspace }}/external/contract-ledger/docker/virtual.Dockerfile
+          context: ${{ github.workspace }}/external/contract-ledger
+          push: true
+          build-args: SCITT_VERSION_OVERRIDE=${{ env.SCITT_VERSION_OVERRIDE }}
+          tags: ${{ env.REGISTRY }}/contract-ledger
+
+
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+**/*.onnx
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,6 @@
+[submodule "external/confidential-sidecar-containers"]
+	path = external/confidential-sidecar-containers
+	url = https://github.com/microsoft/confidential-sidecar-containers
+[submodule "external/contract-ledger"]
+	path = external/contract-ledger
+	url = https://github.com/kapilvgit/contract-ledger.git