From 68fc3b7ad000f269b8f0eb8dc55d4f5adc496b4f Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Mon, 16 Dec 2019 17:10:40 +0800 Subject: [PATCH] Add Prow jobs to automate the release of Docker images and Python SDK (#369) * Add CI script for push to master and release tag. - Publish Python SDK to pypi - Publish Docker images - Publish Helm chart Also update Prow config for test-infra with image tag: v20191211 https://github.com/kubernetes/test-infra/tree/821a860177455dab60160c7624950c2da5c092ed/prow/cluster * Install Python SDK with editable mode during testing So that setuptools_scm does not complain about missing version * Use editable mode for Python SDK installation during testing * Use consistent separator, dash instead of underscore, in script names and options * Remove duplicate README.md in sdk/python folder Reuse the same README in repository root * Push docker images with commit SHA as the tag as well, upon push to master branch --- .prow/config.yaml | 197 +++++++++++++++--- .prow/scripts/download-maven-cache.sh | 2 +- ...oud_sdk.sh => install-google-cloud-sdk.sh} | 4 +- .prow/scripts/publish-docker-image.sh | 51 +++++ .prow/scripts/publish-python-sdk.sh | 47 +++++ .prow/scripts/sync-helm-charts.sh | 4 +- .prow/scripts/test-end-to-end-batch.sh | 4 +- .prow/scripts/test-end-to-end.sh | 2 +- .prow/scripts/test-python-sdk.sh | 2 +- sdk/python/feast/__init__.py | 7 + sdk/python/setup.py | 17 +- 11 files changed, 297 insertions(+), 40 deletions(-) rename .prow/scripts/{install_google_cloud_sdk.sh => install-google-cloud-sdk.sh} (92%) create mode 100755 .prow/scripts/publish-docker-image.sh create mode 100755 .prow/scripts/publish-python-sdk.sh diff --git a/.prow/config.yaml b/.prow/config.yaml index cf765a9903..41f95180fb 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -2,32 +2,41 @@ prowjob_namespace: default pod_namespace: test-pods plank: - allow_cancellations: true - job_url_prefix: http://prow.feast.ai/view/gcs + job_url_prefix_config: + "*": http://prow.feast.ai/view/gcs report_template: '[Full PR test history](https://prow.feast.ai/pr-history?org={{.Spec.Refs.Org}}&repo={{.Spec.Refs.Repo}}&pr={{with index .Spec.Refs.Pulls 0}}{{.Number}}{{end}})' - default_decoration_config: - timeout: 7200000000000 # 2h - grace_period: 15000000000 # 15s - utility_images: - clonerefs: gcr.io/k8s-prow/clonerefs:v20190221-d14461a - initupload: gcr.io/k8s-prow/initupload:v20190221-d14461a - entrypoint: gcr.io/k8s-prow/entrypoint:v20190221-d14461a - sidecar: gcr.io/k8s-prow/sidecar:v20190221-d14461a - gcs_configuration: - bucket: feast-templocation-kf-feast - path_strategy: explicit - gcs_credentials_secret: prow-service-account + pod_pending_timeout: 60m + default_decoration_configs: + "*": + timeout: 1h + grace_period: 15s + utility_images: + clonerefs: gcr.io/k8s-prow/clonerefs:v20190221-d14461a + initupload: gcr.io/k8s-prow/initupload:v20190221-d14461a + entrypoint: gcr.io/k8s-prow/entrypoint:v20190221-d14461a + sidecar: gcr.io/k8s-prow/sidecar:v20190221-d14461a + gcs_configuration: + bucket: feast-templocation-kf-feast + path_strategy: explicit + gcs_credentials_secret: prow-service-account deck: tide_update_period: 1s spyglass: - size_limit: 50e+6 # 50MB - viewers: - "started.json|finished.json": ["metadata"] - "build-log.txt": ["buildlog"] - "report.xml": ["junit"] - "artifacts/.*\\.xml": ["junit"] - "surefire-reports/.*\\.xml": ["junit"] + size_limit: 10e+6 # 10MB + lenses: + - lens: + name: metadata + required_files: + - started.json|finished.json + - lens: + name: buildlog + required_files: + - build-log.txt + - lens: + name: junit + required_files: + - artifacts/.*\.xml tide: queries: @@ -47,7 +56,9 @@ tide: blocker_label: merge-blocker squash_label: tide/squash -# presubmits list Prow jobs that run on pull requests +# presubmits and postsubmits configure ProwJobs: +# https://github.com/kubernetes/test-infra/blob/6571843b1aa7bd6cf577a7a8b9e9971241f424d5/prow/jobs.md + presubmits: gojek/feast: - name: test-core-and-ingestion @@ -131,8 +142,140 @@ presubmits: - name: service-account mountPath: "/etc/service-account" -# TODO: do a release when a git tag is pushed -# -# postsubmits list Prow jobs that run on every push -# postsubmits: -# gojek/feast: \ No newline at end of file +postsubmits: + gojek/feast: + - name: publish-python-sdk + decorate: true + spec: + containers: + - image: python:3 + command: + - sh + - -c + - | + .prow/scripts/publish-python-sdk.sh \ + --directory-path sdk/python --repository pypi + volumeMounts: + - name: pypirc + mountPath: /root/.pypirc + subPath: .pypirc + readOnly: true + volumes: + - name: pypirc + secret: + secretName: pypirc + branches: + # Filter on tags with semantic versioning, prefixed with "v" + # https://github.com/semver/semver/issues/232 + - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ + + - name: publish-docker-images + decorate: true + spec: + containers: + - image: google/cloud-sdk:273.0.0 + command: + - bash + - -c + - | + .prow/scripts/download-maven-cache.sh \ + --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \ + --output-dir $PWD/ + + if [ $PULL_BASE_REF == "master" ]; then + + .prow/scripts/publish-docker-image.sh \ + --repository gcr.io/kf-feast/feast-core \ + --tag dev \ + --file infra/docker/core/Dockerfile \ + --google-service-account-file /etc/gcloud/service-account.json + + .prow/scripts/publish-docker-image.sh \ + --repository gcr.io/kf-feast/feast-serving \ + --tag dev \ + --file infra/docker/serving/Dockerfile \ + --google-service-account-file /etc/gcloud/service-account.json + + docker tag gcr.io/kf-feast/feast-core:dev gcr.io/kf-feast/feast-core:${PULL_BASE_SHA} + docker push gcr.io/kf-feast/feast-core:${PULL_BASE_SHA} + + docker tag gcr.io/kf-feast/feast-serving:dev gcr.io/kf-feast/feast-serving:${PULL_BASE_SHA} + docker push gcr.io/kf-feast/feast-serving:${PULL_BASE_SHA} + + else + + .prow/scripts/publish-docker-image.sh \ + --repository gcr.io/kf-feast/feast-core \ + --tag ${PULL_BASE_REF:1} \ + --file infra/docker/core/Dockerfile \ + --google-service-account-file /etc/gcloud/service-account.json + + .prow/scripts/publish-docker-image.sh \ + --repository gcr.io/kf-feast/feast-serving \ + --tag ${PULL_BASE_REF:1} \ + --file infra/docker/serving/Dockerfile \ + --google-service-account-file /etc/gcloud/service-account.json + + docker tag gcr.io/kf-feast/feast-core:${PULL_BASE_REF:1} gcr.io/kf-feast/feast-core:latest + docker push gcr.io/kf-feast/feast-core:latest + + docker tag gcr.io/kf-feast/feast-serving:${PULL_BASE_REF:1} gcr.io/kf-feast/feast-serving:latest + docker push gcr.io/kf-feast/feast-serving:latest + + fi + volumeMounts: + - name: docker-socket + mountPath: /var/run/docker.sock + - name: service-account + mountPath: /etc/gcloud/service-account.json + subPath: service-account.json + readOnly: true + securityContext: + privileged: true + volumes: + - name: docker-socket + hostPath: + path: /var/run/docker.sock + - name: service-account + secret: + secretName: feast-service-account + branches: + - ^master$ + - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ + + - name: publish-helm-chart + decorate: true + spec: + containers: + - image: google/cloud-sdk:273.0.0-slim + command: + - bash + - -c + - | + gcloud auth activate-service-account --key-file /etc/gcloud/service-account.json + + curl -s https://get.helm.sh/helm-v2.16.1-linux-amd64.tar.gz | tar -C /tmp -xz + mv /tmp/linux-amd64/helm /usr/bin/helm + helm init --client-only + + sed -i "/version: /c\version: ${PULL_BASE_REF:1}" infra/charts/feast/Chart.yaml + sed -i "/ version: /c\ version: ${PULL_BASE_REF:1}" infra/charts/feast/requirements.yaml + + sed -i "/version: /c\version: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-core/Chart.yaml + sed -i "/ tag: /c\ tag: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-core/values.yaml + + sed -i "/version: /c\version: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-serving/Chart.yaml + sed -i "/ tag: /c\ tag: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-serving/values.yaml + + .prow/scripts/sync-helm-charts.sh + volumeMounts: + - name: service-account + mountPath: /etc/gcloud/service-account.json + subPath: service-account.json + readOnly: true + volumes: + - name: service-account + secret: + secretName: feast-service-account + branches: + - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ diff --git a/.prow/scripts/download-maven-cache.sh b/.prow/scripts/download-maven-cache.sh index c9704878e0..91f2dfb301 100755 --- a/.prow/scripts/download-maven-cache.sh +++ b/.prow/scripts/download-maven-cache.sh @@ -26,7 +26,7 @@ if [[ ! ${OUTPUT_DIR} ]]; then usage; exit 1; fi # Install Google Cloud SDK if gsutil command not exists if [[ ! $(command -v gsutil) ]]; then CURRENT_DIR=$(dirname "$BASH_SOURCE") - . "${CURRENT_DIR}"/install_google_cloud_sdk.sh + . "${CURRENT_DIR}"/install-google-cloud-sdk.sh fi gsutil -q cp ${ARCHIVE_URI} /tmp/.m2.tar diff --git a/.prow/scripts/install_google_cloud_sdk.sh b/.prow/scripts/install-google-cloud-sdk.sh similarity index 92% rename from .prow/scripts/install_google_cloud_sdk.sh rename to .prow/scripts/install-google-cloud-sdk.sh index c6356557ce..9d86a55fd2 100755 --- a/.prow/scripts/install_google_cloud_sdk.sh +++ b/.prow/scripts/install-google-cloud-sdk.sh @@ -3,10 +3,10 @@ set -e usage() { - echo "usage: . install_google_cloud_sdk.sh + echo "usage: . install-google-cloud-sdk.sh [--with-key-file local file path to service account json] -NOTE: requires 'dot' before install_google_cloud_sdk.sh +NOTE: requires 'dot' before install-google-cloud-sdk.sh so that the PATH variable is exported succesfully to the calling process, i.e. you don't need to provide full path to gcloud command after installation diff --git a/.prow/scripts/publish-docker-image.sh b/.prow/scripts/publish-docker-image.sh new file mode 100755 index 0000000000..839ee86c6f --- /dev/null +++ b/.prow/scripts/publish-docker-image.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +usage() +{ + echo "usage: publish-docker-image.sh + + --repository the target repository to upload the Docker image, example: + gcr.io/kf-feast/feast-core + + --tag the tag for the Docker image, example: 1.0.4 + + --file path to the Dockerfile + + [--google-service-account-file + path to Google Cloud service account JSON key file] +" +} + +while [ "$1" != "" ]; do + case "$1" in + --repository ) REPOSITORY="$2"; shift;; + --tag ) TAG="$2"; shift;; + --file ) FILE="$2"; shift;; + --google-service-account-file ) GOOGLE_SERVICE_ACCOUNT_FILE="$2"; shift;; + -h | --help ) usage; exit;; + * ) usage; exit 1 + esac + shift +done + +if [ -z $REPOSITORY ]; then usage; exit 1; fi +if [ -z $TAG ]; then usage; exit 1; fi +if [ -z $FILE ]; then usage; exit 1; fi + +if [ $GOOGLE_SERVICE_ACCOUNT_FILE ]; then + gcloud -q auth activate-service-account --key-file $GOOGLE_SERVICE_ACCOUNT_FILE + gcloud -q auth configure-docker +fi + +echo "============================================================" +echo "Building Docker image $REPOSITORY:$TAG" +echo "============================================================" +docker build -t $REPOSITORY:$TAG --build-arg REVISION=$TAG -f $FILE . + +echo "============================================================" +echo "Pushing Docker image $REPOSITORY:$TAG" +echo "============================================================" +docker push $REPOSITORY:$TAG diff --git a/.prow/scripts/publish-python-sdk.sh b/.prow/scripts/publish-python-sdk.sh new file mode 100755 index 0000000000..582d9072b2 --- /dev/null +++ b/.prow/scripts/publish-python-sdk.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +usage() +{ + echo "usage: publish-python-sdk.sh + + --directory-path absolute path to the python package, this directory + should contain 'setup.py' file + + --repository the repository name where the package will be uploaded, + check your .pypirc configuration file for the list of + valid repositories, usually it's 'pypi' or 'testpypi' +" +} + +while [ "$1" != "" ]; do + case "$1" in + --directory-path ) DIRECTORY_PATH="$2"; shift;; + --repository ) REPOSITORY="$2"; shift;; + -h | --help ) usage; exit;; + * ) usage; exit 1 + esac + shift +done + +if [ -z $DIRECTORY_PATH ]; then usage; exit 1; fi +if [ -z $REPOSITORY ]; then usage; exit 1; fi + +ORIGINAL_DIR=$PWD +cd $DIRECTORY_PATH + +echo "============================================================" +echo "Generating distribution archives" +echo "============================================================" +python3 -m pip install --user --upgrade setuptools wheel +python3 setup.py sdist bdist_wheel + +echo "============================================================" +echo "Uploading distribution archives" +echo "============================================================" +python3 -m pip install --user --upgrade twine +python3 -m twine upload --repository $REPOSITORY dist/* + +cd $ORIGINAL_DIR diff --git a/.prow/scripts/sync-helm-charts.sh b/.prow/scripts/sync-helm-charts.sh index 918979dd3f..88fc04f7d1 100755 --- a/.prow/scripts/sync-helm-charts.sh +++ b/.prow/scripts/sync-helm-charts.sh @@ -46,7 +46,7 @@ if helm repo index --url "$repo_url" --merge "$index_dir/index.yaml" "$sync_dir" gsutil -m rsync "$sync_dir" "$bucket" # Make sure index.yaml is synced last - gsutil cp "$index_dir/index.yaml" "$bucket" + gsutil -h "Cache-Control:no-cache,max-age=0" cp "$index_dir/index.yaml" "$bucket" else log_error "Exiting because unable to update index. Not safe to push update." exit 1 @@ -54,4 +54,4 @@ fi ls -l "$sync_dir" -exit "$exit_code" \ No newline at end of file +exit "$exit_code" diff --git a/.prow/scripts/test-end-to-end-batch.sh b/.prow/scripts/test-end-to-end-batch.sh index 82d6286027..b370c5b045 100755 --- a/.prow/scripts/test-end-to-end-batch.sh +++ b/.prow/scripts/test-end-to-end-batch.sh @@ -28,7 +28,7 @@ Installing gcloud SDK " if [[ ! $(command -v gsutil) ]]; then CURRENT_DIR=$(dirname "$BASH_SOURCE") - . "${CURRENT_DIR}"/install_google_cloud_sdk.sh + . "${CURRENT_DIR}"/install-google-cloud-sdk.sh fi export GOOGLE_APPLICATION_CREDENTIALS=/etc/service-account/service-account.json @@ -210,7 +210,7 @@ bash /tmp/miniconda.sh -b -p /root/miniconda -f source ~/.bashrc # Install Feast Python SDK and test requirements -pip install -q sdk/python +pip install -qe sdk/python pip install -qr tests/e2e/requirements.txt echo " diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh index 96d9ff60a1..2c6f4a098f 100755 --- a/.prow/scripts/test-end-to-end.sh +++ b/.prow/scripts/test-end-to-end.sh @@ -191,7 +191,7 @@ bash /tmp/miniconda.sh -b -p /root/miniconda -f source ~/.bashrc # Install Feast Python SDK and test requirements -pip install -q sdk/python +pip install -qe sdk/python pip install -qr tests/e2e/requirements.txt echo " diff --git a/.prow/scripts/test-python-sdk.sh b/.prow/scripts/test-python-sdk.sh index 86b88f6082..e7e1cc2487 100755 --- a/.prow/scripts/test-python-sdk.sh +++ b/.prow/scripts/test-python-sdk.sh @@ -7,5 +7,5 @@ LOGS_ARTIFACT_PATH=/logs/artifacts cd sdk/python pip install -r requirements-ci.txt -pip install . +pip install -e . pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml diff --git a/sdk/python/feast/__init__.py b/sdk/python/feast/__init__.py index bcd714d32b..5f62ab496f 100644 --- a/sdk/python/feast/__init__.py +++ b/sdk/python/feast/__init__.py @@ -1,3 +1,10 @@ +from pkg_resources import get_distribution, DistributionNotFound +try: + __version__ = get_distribution(__name__).version +except DistributionNotFound: + # package is not installed + pass + from .client import Client from .entity import Entity from .feature_set import FeatureSet diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 3edb3c14bd..66cad904b0 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -12,14 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + from setuptools import find_packages, setup NAME = "feast" -DESCRIPTION = "Python sdk for Feast" +DESCRIPTION = "Python SDK for Feast" URL = "https://github.com/gojek/feast" AUTHOR = "Feast" REQUIRES_PYTHON = ">=3.6.0" -VERSION = "0.3.2" REQUIRED = [ "Click==7.*", @@ -45,11 +46,17 @@ "google", ] +# README file from Feast repo root directory +README_FILE = os.path.join(os.path.dirname(__file__), "..", "..", "README.md") +with open(os.path.join(README_FILE), "r") as f: + LONG_DESCRIPTION = f.read() + setup( name=NAME, - version=VERSION, - description=DESCRIPTION, author=AUTHOR, + description=DESCRIPTION, + long_description=LONG_DESCRIPTION, + long_description_content_type="text/markdown", python_requires=REQUIRES_PYTHON, url=URL, packages=find_packages(exclude=("tests",)), @@ -68,4 +75,6 @@ "Programming Language :: Python :: 3.6", ], entry_points={"console_scripts": ["feast=cli:cli"]}, + use_scm_version={"root": "../..", "relative_to": __file__}, + setup_requires=["setuptools_scm"], )