feat: add codebase for model grpc service

.air.toml

@@ -0,0 +1,32 @@
+root = "."
+tmp_dir = "tmp"
+
+[build]
+  bin = "./tmp/main"
+  cmd = "go build -a -o ./tmp/main ./cmd"
+  delay = 1000
+  exclude_dir = ["assets", "tmp", "vendor"]
+  exclude_file = []
+  exclude_regex = []
+  exclude_unchanged = false
+  follow_symlink = false
+  full_bin = ""
+  include_dir = []
+  include_ext = ["go", "tpl", "tmpl", "html"]
+  kill_delay = "0s"
+  log = "build-errors.log"
+  send_interrupt = false
+  stop_on_error = true
+
+[color]
+  app = ""
+  build = "yellow"
+  main = "magenta"
+  runner = "green"
+  watcher = "cyan"
+
+[log]
+  time = false
+
+[misc]
+  clean_on_exit = false

.github/pull_request_template.md

@@ -0,0 +1,24 @@
+# What problem does this PR solve?
+
+# How is the solution implemented?
+
+- [ ] **This PR depends on other PRs that are not merged yet**: [link to PRs]
+
+# Where should the reviewer start?
+
+# Checklist
+
+Author and reviewer must actively check the following points and tick the corresponding box.
+
+**The PR is rebased on the latest commit of the relevant target branch**
+
+- [ ] Author
+- [ ] Reviewer
+
+**The PR commits**
+
+- **follow the [7 rules of commit messages](https://chris.beams.io/posts/git-commit/)**
+- **follow the [conventional commits guidelines](https://www.conventionalcommits.org/)**
+- **are reordered to squash trivial commits together (use [git rebase](http://gitready.com/advanced/2009/03/20/reorder-commits-with-rebase.html))**
+  - [ ] Author
+  - [ ] Reviewer

.github/semantic.yml

@@ -0,0 +1,2 @@
+# Always validate the PR title AND all the commits
+titleAndCommits: true

.github/workflows/cd.yml

@@ -0,0 +1,204 @@
+name: cd
+
+on:
+  push:
+    branches:
+      - main
+
+env:
+  GAR_REGISTRY: europe-west2-docker.pkg.dev/prj-c-devops-artifacts-a306
+  GAR_REPOSITORY: model
+  GAR_IMAGE: model-backend
+  OPENAPI_NAME: model
+
+jobs:
+  get-version:
+    runs-on: [self-hosted, on-prem, dgx]
+    outputs:
+      CORE_VERSION: ${{ steps.get-version.outputs.CORE_VERSION }}
+      GITHUB_SHA: ${{ steps.get-version.outputs.GITHUB_SHA }}
+    steps:
+      - name: Checkout (latest)
+        uses: actions/checkout@v2
+
+      - name: Get version
+        id: get-version
+        shell: bash
+        run: |
+          CORE_VERSION=$(cat version.txt)
+          GITHUB_SHA=$(echo ${{ github.sha }} | cut -c -7)
+          echo "::set-output name=CORE_VERSION::$CORE_VERSION"
+          echo "::set-output name=GITHUB_SHA::$GITHUB_SHA"
+  release:
+    runs-on: [self-hosted, on-prem, dgx]
+    outputs:
+      RELEASE_CREATED: ${{ steps.release.outputs.release_created }}
+      TAG_NAME: ${{ steps.release.outputs.tag_name }}
+      SHA: ${{ steps.release.outputs.sha }}
+    steps:
+      - name: Import Vault secrets
+        id: vault
+        uses: hashicorp/[email protected]
+        with:
+          url: https://vault.instill.tech
+          method: approle
+          roleId: ${{ secrets.vaultAppRoleRoleId }}
+          secretId: ${{ secrets.vaultAppRoleSecretId }}
+          tlsSkipVerify: false
+          extraHeaders: |
+            CF-Access-Client-Id: ${{ secrets.cfAccessClientId }}
+            CF-Access-Client-Secret: ${{ secrets.cfAccessClientSecret }}
+          secrets: |
+            secret/data/devops/github/[email protected]/github-action personal-access-token | GITHUB_TOKEN ;
+      - name: Release please action
+        id: release
+        uses: GoogleCloudPlatform/[email protected]
+        with:
+          # if we use the repo default "GITHUB_TOKEN", the release PR opened by release-please-action (i.e., by GitHub Action workflow) won't trigger push event when it is merged again to main.
+          token: ${{ steps.vault.outputs.GITHUB_TOKEN }}
+          release-type: simple
+          bump-minor-pre-major: true
+          bump-patch-for-minor-pre-major: true
+          package-name: ${{ github.event.repository.name }}
+          changelog-types: '[{"type":"feat","section":"Features","hidden":false}, {"type":"fix","section":"Bug Fixes","hidden":false}, {"type":"chore","section":"Miscellaneous","hidden":false}, {"type":"ci","section":"Miscellaneous","hidden":false}, {"type":"refactor","section":"Refactor","hidden":false}]'
+
+  build:
+    runs-on: [self-hosted, on-prem, dgx]
+    needs: [get-version, release]
+    steps:
+      - name: Checkout (latest)
+        uses: actions/checkout@v2
+
+      - name: Build image
+        run: |
+          if [[ "${{ needs.release.outputs.RELEASE_CREATED }}" == "true" ]]; then
+            echo "Released - Tag container with version core"
+            DOCKER_BUILDKIT=1 docker build -t $GAR_REGISTRY/$GAR_REPOSITORY/$GAR_IMAGE:${{ needs.get-version.outputs.CORE_VERSION }} .
+          else
+            echo "Non-released - Tag container with version core and build metadata"
+            DOCKER_BUILDKIT=1 docker build -t $GAR_REGISTRY/$GAR_REPOSITORY/$GAR_IMAGE:${{ needs.get-version.outputs.CORE_VERSION }}_${{ needs.get-version.outputs.GITHUB_SHA }} .
+          fi
+  push:
+    runs-on: [self-hosted, on-prem, dgx]
+    needs: [release, get-version, build]
+    steps:
+      - name: Import Vault secrets
+        id: vault
+        uses: hashicorp/[email protected]
+        with:
+          url: https://vault.instill.tech
+          method: approle
+          roleId: ${{ secrets.vaultAppRoleRoleId }}
+          secretId: ${{ secrets.vaultAppRoleSecretId }}
+          tlsSkipVerify: false
+          extraHeaders: |
+            CF-Access-Client-Id: ${{ secrets.cfAccessClientId }}
+            CF-Access-Client-Secret: ${{ secrets.cfAccessClientSecret }}
+          # A magic json output trick: https://github.com/hashicorp/vault-action/issues/153#issuecomment-824329289
+          secrets: |
+            secret/data/devops/gcp/[email protected] $.$ | GAR_SA_KEY;
+      - name: Push image
+        run: |
+          echo $GAR_SA_KEY | docker login -u _json_key --password-stdin https://$GAR_REGISTRY
+          if [[ "${{ needs.release.outputs.RELEASE_CREATED }}" == "true" ]]; then
+            echo "Released - Tag container with version core"
+            docker push $GAR_REGISTRY/$GAR_REPOSITORY/$GAR_IMAGE:${{ needs.get-version.outputs.CORE_VERSION }}
+          else
+            echo "Non-released - Tag container with version core and build metadata"
+            docker push $GAR_REGISTRY/$GAR_REPOSITORY/$GAR_IMAGE:${{ needs.get-version.outputs.CORE_VERSION }}_${{ needs.get-version.outputs.GITHUB_SHA }}
+          fi
+  purge:
+    runs-on: [self-hosted, on-prem, dgx]
+    needs: [release, get-version, push]
+    steps:
+      - name: Import Vault secrets
+        id: vault
+        uses: hashicorp/[email protected]
+        with:
+          url: https://vault.instill.tech
+          method: approle
+          roleId: ${{ secrets.vaultAppRoleRoleId }}
+          secretId: ${{ secrets.vaultAppRoleSecretId }}
+          tlsSkipVerify: false
+          extraHeaders: |
+            CF-Access-Client-Id: ${{ secrets.cfAccessClientId }}
+            CF-Access-Client-Secret: ${{ secrets.cfAccessClientSecret }}
+          # A magic json output trick: https://github.com/hashicorp/vault-action/issues/153#issuecomment-824329289
+          secrets: |
+            secret/data/devops/gcp/[email protected] $.$ | GAR_SA_KEY;
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@master
+        with:
+          version: "319.0.0"
+          service_account_key: ${{ steps.vault.outputs.GAR_SA_KEY }}
+          export_default_credentials: true
+
+      - name: Purge build-metadata images
+        shell: bash
+        run: |
+          tags=$(gcloud artifacts docker images list $GAR_REGISTRY/$GAR_REPOSITORY/$GAR_IMAGE --include-tags --filter='tags~"((0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)_\w+)"' --format="value(TAGS)" | tr ',' ' ')
+          if [[ "${{ needs.release.outputs.RELEASE_CREATED }}" == "true" ]]; then
+            echo "Released - purge all build-metadata images"
+            for tag in $tags; do
+              if [[ "$tag" == "${{ needs.get-version.outputs.CORE_VERSION }}" ]] || [[ "$tag" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then
+                echo "Image tag to keep: $tag"
+              else
+                echo "Image tag to purge: $tag"
+                gcloud artifacts docker tags delete $GAR_REGISTRY/$GAR_REPOSITORY/$GAR_IMAGE:"$tag" --quiet
+              fi
+            done
+          else
+            echo "Non-released - preserve the latest commit image in main branch and purge all the others"
+            for tag in $tags; do
+              if [ "$tag" = "${{ needs.get-version.outputs.CORE_VERSION }}_${{ needs.get-version.outputs.GITHUB_SHA }}" ] || [ "$tag" = "${{ needs.get-version.outputs.CORE_VERSION }}" ]; then
+                echo "Image tag to keep: $tag"
+              else
+                echo "Image tag to purge: $tag"
+                gcloud artifacts docker tags delete $GAR_REGISTRY/$GAR_REPOSITORY/$GAR_IMAGE:"$tag" --quiet
+              fi
+            done
+          fi
+  # Only tag and upload OpenAPI spec at release
+  release-public:
+    runs-on: [self-hosted, on-prem, dgx]
+    needs: release
+    if: ${{ needs.release.outputs.RELEASE_CREATED }}
+    steps:
+      - name: Checkout (latest)
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # fetch all history for git commands in below steps
+
+      - name: Create public tag
+        shell: bash
+        run: |
+          SHA_LAST_CHANGED=$(git --no-pager log -n 1 --pretty=format:%H -- version-public.txt)
+          echo "Last version-public.txt changed occurred in: $SHA_LAST_CHANGED"
+          DIFF=$(git --no-pager diff HEAD~1 -- CHANGELOG.md)
+          if [ -n "$DIFF" ]
+          then
+              if [[ $DIFF =~ $SHA_LAST_CHANGED ]]
+              then
+                  echo "version-public.txt was changed in this release"
+                  TAG=v$(cat version-public.txt)-public
+                  git config user.name droplet-bot
+                  git config user.email [email protected]
+                  git tag -a "$TAG" -m "Public release $TAG for code version ${{ needs.release.outputs.TAG_NAME }}"
+                  git push origin "$TAG"
+              else
+                  echo "version-public.txt was not changed in this release"
+              fi
+          else
+              echo "No changes recorded in CHANGELOG.md (no version was released)"
+          fi
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@master
+        with:
+          service_account_key: ${{ secrets.googleTerraformSaKey }} # vault-action does not support json output yet so we store this as an organization secret
+          export_default_credentials: true
+
+      - name: Upload openapi file to GCS bucket
+        shell: bash
+        run: |
+          echo "Uploaded openapi_$TAG.yaml to GCS bucket"
+          gsutil cp api/openapi.yaml gs://public-europe-west2-c-artifacts/docs/api/$OPENAPI_NAME/openapi_v$(cat version-public.txt).yaml

.github/workflows/ci.yml

@@ -0,0 +1,68 @@
+name: ci
+
+on:
+  pull_request:
+    branches:
+      - main
+
+env:
+  GAR_REGISTRY: europe-west2-docker.pkg.dev/prj-c-devops-artifacts-a306
+  GAR_REPOSITORY: model
+  GAR_IMAGE: model-backend
+
+jobs:
+  get-version:
+    runs-on: [self-hosted, on-prem, dgx]
+    # Skip if the event is triggered by the PR of release-please action (for pushing back to main branch for updating changelog and bumping up version)
+    if: ${{ !contains(github.head_ref, 'release') }}
+    outputs:
+      CORE_VERSION: ${{ steps.get-version.outputs.CORE_VERSION }}
+      GITHUB_SHA: ${{ steps.get-version.outputs.GITHUB_SHA }}
+    steps:
+      - name: Checkout (latest)
+        uses: actions/checkout@v2
+
+      - name: Get version
+        id: get-version
+        shell: bash
+        run: |
+          CORE_VERSION=$(cat version.txt)
+          GITHUB_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c -7)
+          echo "::set-output name=CORE_VERSION::$CORE_VERSION"
+          echo "::set-output name=GITHUB_SHA::$GITHUB_SHA"
+
+  build:
+    runs-on: [self-hosted, on-prem, dgx]
+    needs: [get-version]
+    steps:
+      - name: Checkout (latest)
+        uses: actions/checkout@v2
+
+      - name: Build build-metadata image
+        run: |
+          DOCKER_BUILDKIT=1 docker build -t $GAR_REGISTRY/$GAR_REPOSITORY/$GAR_IMAGE:${{ needs.get-version.outputs.CORE_VERSION }}_${{ needs.get-version.outputs.GITHUB_SHA }} .
+
+  push:
+    runs-on: [self-hosted, on-prem, dgx]
+    needs: [get-version, build]
+    steps:
+      - name: Import Vault secrets
+        id: vault
+        uses: hashicorp/[email protected]
+        with:
+          url: https://vault.instill.tech
+          method: approle
+          roleId: ${{ secrets.vaultAppRoleRoleId }}
+          secretId: ${{ secrets.vaultAppRoleSecretId }}
+          tlsSkipVerify: false
+          extraHeaders: |
+            CF-Access-Client-Id: ${{ secrets.cfAccessClientId }}
+            CF-Access-Client-Secret: ${{ secrets.cfAccessClientSecret }}
+          # A magic json output trick: https://github.com/hashicorp/vault-action/issues/153#issuecomment-824329289
+          secrets: |
+            secret/data/devops/gcp/[email protected] $.$ | GAR_SA_KEY;
+
+      - name: Push build-metadata image
+        run: |
+          echo $GAR_SA_KEY | docker login -u _json_key --password-stdin https://$GAR_REGISTRY
+          docker push $GAR_REGISTRY/$GAR_REPOSITORY/$GAR_IMAGE:${{ needs.get-version.outputs.CORE_VERSION }}_${{ needs.get-version.outputs.GITHUB_SHA }}

CODEOWNERS

@@ -0,0 +1 @@
+* @phelan164

Dockerfile

@@ -0,0 +1,51 @@
+FROM golang:1.17.2 AS build
+WORKDIR /go/src
+
+COPY go.mod .
+COPY go.sum .
+
+ENV CGO_ENABLED=0
+RUN go mod download
+
+COPY . /go/src
+
+RUN --mount=type=cache,target=/root/.cache/go-build go build -a -o /openapi ./cmd
+RUN --mount=type=cache,target=/root/.cache/go-build go build -a -o /migrate ./migrate
+
+# ! DO NOT USE THIS SELF-SIGNED CERTIFICATE ON PRODUCTION
+# Generate self-signed SSL/TLS certificate
+WORKDIR /model-backend/ssl
+# Create a private key ca.key
+RUN openssl genrsa -out ca.key 2048
+# Create a self-signed root Certificate Authority (root-CA) certificate ca.crt
+RUN openssl req -new -x509 -days 365 -key ca.key -subj "/C=UK/ST=London/L=London/O=Instill AI/CN=localhost" -out ca.crt
+# Create a Certificate Signing Request (CSR) tls.csr and its corresponding private key
+RUN openssl req -newkey rsa:2048 -nodes -keyout tls.key -subj "/C=UK/ST=London/L=London/O=Instill AI/CN=localhost" -out tls.csr
+# Convert a CSR into a self-signed certificate using extensions for a CA
+RUN echo subjectAltName=DNS:localhost > extfile.cnf
+RUN openssl x509 -req -extfile extfile.cnf -days 365 -in tls.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out tls.crt
+RUN rm extfile.cnf
+
+# Install the Root CA certificates
+RUN cp ca.crt /etc/ssl/certs/ca.crt && update-ca-certificates
+
+#
+#
+# Prod image
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4 AS runtime
+# FROM gcr.io/distroless/base AS runtime
+
+WORKDIR /model-backend
+
+ENV GIN_MODE=release
+
+COPY --from=build /openapi ./
+COPY --from=build /go/src/.env ./
+COPY --from=build /go/src/configs ./configs/
+COPY --from=build /go/src/pkg/db/migrations ./pkg/db/migrations/
+COPY --from=build /migrate ./
+
+COPY --from=build /model-backend ssl
+COPY --from=build /model-backend /ssl/certs/ca.crt
+
+ENTRYPOINT ["./openapi"]