introduce docker container production (#660)

Introduces the production of docker containers as a CI step.
Currently only provides a rolling-release version that builds
on every push to main.  Images are deployed to ghcr.io.

The PR includes two variations on building the images.  We'll
likely only want to stick with one or the other.

.dockerignore

@@ -0,0 +1,3 @@
+.git
+.gitignore
+.dockerignore

.github/workflows/container.yml

@@ -0,0 +1,114 @@
+name: Publish Docker Container Images
+on:
+  push:
+    branches: [main]
+
+env:
+  REGISTRY: ghcr.io
+  REPO_NAME: ${{ github.repository }}
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+
+  # ------------------------------------------------------------------------------------------
+  # To be decided: Script-Deploy or Dockerfile-Deploy:
+  # Script:
+  #   + Separates the golang build from the corso build.
+  #   - Haven't figured out multiplatform builds yet.
+  #   - Doesn't cache, always takes 10-15 minutes per build in the matrix.
+  # Dockerfile:
+  #   + Once cached, takes <1m to deploy.
+  #   + Multiplatform.
+  #   + Extended features (such as tagging) can be handled by more github actions.
+  #   - When not cached, can take >2 hours to build (at least initially).
+  #   - Currently includes the complete golang:1.18 image.
+  # ------------------------------------------------------------------------------------------
+
+  Script-Deploy:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: build
+    strategy:
+      matrix:
+        BUILD_ARCH: [amd64, arm64]
+        BUILD_OS: [linux]
+    env:
+      IMAGE_PREFIX: ghcr.io
+      VERSION_SUFFIX: rolling
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Run build script
+        run: >
+          ./build-container.sh
+          --arch ${{ matrix.BUILD_ARCH }}
+          --prefix ${{ env.IMAGE_PREFIX }}
+          --suffix ${{ env.VERSION_SUFFIX }}
+
+      # login step boilerplate from:
+      # https://docs.github.com/en/packages/managing-github-packages-using-github-actions-workflows/publishing-and-installing-a-package-with-github-actions#upgrading-a-workflow-that-accesses-ghcrio
+      - name: Log in to registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
+
+      - name: Push image
+        env:
+          IMAGE_ID: ${{ env.IMAGE_PREFIX }}/alcionai/corso
+          VERSION: ${{ matrix.BUILD_OS }}-${{ matrix.BUILD_ARCH }}-${{ env.VERSION_SUFFIX }}
+        run: |
+          docker images -a
+          docker push ${{ env.IMAGE_ID }}:${{ env.VERSION }}
+
+  Dockerfile-Deploy:
+    runs-on: ubuntu-latest
+    env:
+      TARGETOS: linux
+      TARGETARCH: arm64
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # apparently everyone uses this step
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      # setup Docker buld action
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      # In case we want to switch to dockerhub      
+      # - name: Login to DockerHub
+      #   uses: docker/login-action@v2
+      #   with:
+      #     username: ${{ secrets.DOCKERHUB_USERNAME }}
+      #     password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # retrieve credentials for ghcr.io
+      - name: Login to Github Packages
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # build the image
+      - name: Build image and push to Docker Hub and GitHub Container Registry
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./docker/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ghcr.io/alcionai/corso:rolling
+          # use the github cache
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # check the image digest
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

README.md

@@ -22,7 +22,16 @@ TODO - Link to the appropriate page in the published docs.
 ./build/build-container.sh 
 ```
 
-## Contribution Guidelines
+# Containers
+
+Corso images are hosted on [ghrc.io](https://github.com/alcionai/corso/pkgs/container/corso).
+
+Rolling release
+```sh
+docker pull ghcr.io/alcionai/corso:{SHA} --platform linux/arm64
+```
+
+# Contribution Guidelines
 
 TODO 
 

build/Dockerfile

@@ -0,0 +1,15 @@
+# syntax=docker/dockerfile:1
+
+# This dockerfile is configured to be run by the /corso/build/build-container.sh
+# script.  Using docker to build this file directly will fail.
+
+FROM gcr.io/distroless/base-debian10
+# FROM gcr.io/distroless/base:debug
+
+WORKDIR /
+
+COPY ./bin/corso ./
+
+USER nonroot:nonroot
+
+ENTRYPOINT ["/corso"]

build/build-container.sh

@@ -10,12 +10,15 @@ usage() {
   echo "-----"
   echo "Flags"
   echo "  -h|--help         Help"
-  echo "    |--arm          Set the architecture to arm64 (default: amd64)"
+  echo "  -a|--arch         Set the architecture to the specified value (default: amd64)"
+  echo "  -p|--prefix       Prefixes the image name."
+  echo "  -s|--suffix       Suffixes the version."
   echo " "
   echo "-----"
   echo "Example Usage:"
   echo "  ./build/build-container.sh"
-  echo "  ./build/build-container.sh --arm" 
+  echo "  ./build/build-container.sh --arch arm64"
+  echo "  ./build/build-container.sh --arch arm64 --prefix ghcr.io --suffix nightly"
   echo " "
   exit 0
 }
@@ -25,6 +28,8 @@ PROJECT_ROOT=$(dirname ${SCRIPT_ROOT})
 
 OS=linux
 ARCH=amd64
+IMAGE_NAME_PREFIX=
+IMAGE_TAG_SUFFIX=
 
 while [ "$#" -gt 0 ]
 do
@@ -33,31 +38,52 @@ do
     usage
     exit 0
     ;;
-  --arm)
-    ARCH=arm64
+  -a|--arch)
+    ARCH=$2
+    shift
+    ;;
+  -p|--prefix)
+    IMAGE_NAME_PREFIX=$2
+    shift
+    ;;
+  -s|--suffix)
+    IMAGE_TAG_SUFFIX=$2
+    shift
     ;;
   -*)
-    echo "Invalid option '$1'. Use -h|--help to see the valid options" >&2
+    echo "Invalid flag '$1'. Use -h|--help to see the valid options" >&2
     return 1
     ;;
   *)
-    echo "Invalid option '$1'. Use -h|--help to see the valid options" >&2
+    echo "Invalid arg '$1'. Use -h|--help to see the valid options" >&2
     return 1
     ;;
   esac
   shift
 done
 
-IMAGE_TAG=${OS}-${ARCH}-$(git describe --tags --always --dirty)
+IMAGE_TAG=${OS}-${ARCH}
+if [ ! -z "${IMAGE_TAG_SUFFIX}" ]; then
+  IMAGE_TAG=${IMAGE_TAG}-${IMAGE_TAG_SUFFIX}
+fi
+
 IMAGE_NAME=alcionai/corso:${IMAGE_TAG}
+if [ ! -z "${IMAGE_NAME_PREFIX}" ]; then
+  IMAGE_NAME=${IMAGE_NAME_PREFIX}/${IMAGE_NAME}
+fi
 
 ${SCRIPT_ROOT}/build.sh --arch ${ARCH}
 
-echo "building container"
+echo "-----"
+echo "building corso container ${IMAGE_NAME}"
+echo "-----"
+
 set -x
 docker buildx build --tag ${IMAGE_NAME}     \
   --platform ${OS}/${ARCH}                  \
-  --file ${PROJECT_ROOT}/docker/Dockerfile  \
+  --file ${PROJECT_ROOT}/build/Dockerfile  \
   ${PROJECT_ROOT}
 set +x
-echo "container built successfully ${IMAGE_NAME}"
+
+echo "-----"
+echo "container built successfully"

build/build.sh

@@ -4,12 +4,19 @@ set -e
 
 SCRIPT_ROOT=$(dirname $(readlink -f $0))
 PROJECT_ROOT=$(dirname ${SCRIPT_ROOT})
-SRC_DIR=${PROJECT_ROOT}
-CORSO_BUILD_ARGS=''
 
-CORSO_BUILD_CONTAINER_DIR=/go/src/github.com/alcionai/corso
-CORSO_BUILD_CONTAINER_SRC_DIR=${CORSO_BUILD_CONTAINER_DIR}/src
+CORSO_BUILD_CONTAINER=/go/src/github.com/alcionai/corso
+CORSO_BUILD_CONTAINER_SRC=${CORSO_BUILD_CONTAINER}/src
+CORSO_BUILD_PKG_MOD=/go/pkg/mod
+CORSO_BUILD_TMP=/tmp/.corsobuild
+CORSO_BUILD_TMP_CACHE=${CORSO_BUILD_TMP}/cache
+CORSO_BUILD_TMP_MOD=${CORSO_BUILD_TMP}/mod
+CORSO_CACHE=${CORSO_BUILD_TMP_CACHE}
+CORSO_MOD_CACHE=${CORSO_BUILD_PKG_MOD}/cache
+
+CORSO_BUILD_ARGS=''
 
+GOVER=1.18
 GOOS=linux
 GOARCH=amd64
 
@@ -25,26 +32,31 @@ do
 done
 
 # temporary directory for caching go build
-mkdir -p /tmp/.corsobuild/cache
+mkdir -p ${CORSO_BUILD_TMP_CACHE}
 # temporary directory for caching go modules (needed for fast cross-platform build)
-mkdir -p /tmp/.corsobuild/mod
+mkdir -p ${CORSO_BUILD_TMP_MOD}
 
-echo "building corso"
-set -x
-docker run --rm --mount type=bind,src=${SRC_DIR},dst=${CORSO_BUILD_CONTAINER_DIR}    \
-       --mount type=bind,src=/tmp/.corsobuild/cache,dst=/tmp/.corsobuild/cache       \
-       --mount type=bind,src=/tmp/.corsobuild/mod,dst=/go/pkg/mod                    \
-       --workdir ${CORSO_BUILD_CONTAINER_SRC_DIR}                                    \
-       --env GOCACHE=/tmp/.corsobuild/cache                                          \
-       --env GOOS=${GOOS}                                                            \
-       --env GOARCH=${GOARCH}                                                        \
-       --env GOCACHE=/tmp/.corsobuild/cache                                          \
-       --entrypoint /usr/local/go/bin/go                                             \
-       golang:1.18                                                                   \
-       build ${CORSO_BUILD_ARGS}
+echo "-----"
+echo "building corso binary for ${GOOS}-${GOARCH}"
+echo "-----"
 
-mkdir -p ${PROJECT_ROOT}/bin
+set -x
+docker run --rm \
+  --mount type=bind,src=${PROJECT_ROOT},dst=${CORSO_BUILD_CONTAINER}          \
+  --mount type=bind,src=${CORSO_BUILD_TMP_CACHE},dst=${CORSO_BUILD_TMP_CACHE} \
+  --mount type=bind,src=${CORSO_BUILD_TMP_MOD},dst=${CORSO_BUILD_PKG_MOD}     \
+  --workdir ${CORSO_BUILD_CONTAINER_SRC}                                      \
+  --env GOMODCACHE=${CORSO_MOD_CACHE}                                         \
+  --env GOCACHE=${CORSO_CACHE}                                                \
+  --env GOOS=${GOOS}                                                          \
+  --env GOARCH=${GOARCH}                                                      \
+  --entrypoint /usr/local/go/bin/go                                           \
+  golang:${GOVER}                                                             \
+  build ${CORSO_BUILD_ARGS}
 set +x
 
-echo "creating binary image in bin/corso"
+mkdir -p ${PROJECT_ROOT}/bin
 mv ${PROJECT_ROOT}/src/corso ${PROJECT_ROOT}/bin/corso
+
+echo "-----"
+echo "created binary image in ${PROJECT_ROOT}/bin/corso"

docker/Dockerfile

@@ -1,11 +1,29 @@
 # syntax=docker/dockerfile:1
 
+# This dockerfile is able to make a quick, local image of corso.
+# It is not used for deployments.
+
+## Build
+FROM golang:1.18 AS base
+
+WORKDIR /src
+
+COPY ./src/go.mod .
+COPY ./src/go.sum .
+RUN go mod download
+
+COPY ./src .
+
+FROM base AS build
+ARG TARGETOS
+ARG TARGETARCH
+RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o /corso .
+
+## Deploy
 FROM gcr.io/distroless/base-debian10
 
-WORKDIR /
+COPY --from=build /corso /
 
-COPY ./bin/corso ./
-
 USER nonroot:nonroot
 
-ENTRYPOINT ["/corso"] 
+ENTRYPOINT ["/corso"]

docker/docker-bake.hcl

@@ -0,0 +1,12 @@
+// docker-bake.hcl
+target "docker-metadata-action" {}
+
+target "build" {
+  inherits = ["docker-metadata-action"]
+  context = "./"
+  dockerfile = "Dockerfile"
+  platforms = [
+    "linux/amd64",
+    "linux/arm64",
+  ]
+}