From 4d62e669178800fd2324d895dbe7e73fc1d81464 Mon Sep 17 00:00:00 2001 From: Tsotne Tabidze Date: Fri, 19 Nov 2021 15:43:27 -0800 Subject: [PATCH 1/2] Fix feature server docker image tag generation in pr integration tests Signed-off-by: Tsotne Tabidze --- .github/workflows/pr_integration_tests.yml | 15 ++++++++++++--- sdk/python/feast/constants.py | 3 +++ sdk/python/feast/infra/aws.py | 19 +++++++------------ 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 7699d70381..31533fa29a 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -34,6 +34,9 @@ jobs: - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 + - name: Set ECR image tag + id: image-tag + run: echo '::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`' - name: Build and push env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} @@ -42,9 +45,11 @@ jobs: run: | docker build \ --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ - --tag $ECR_REGISTRY/$ECR_REPOSITORY:`git rev-parse HEAD` \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:`git rev-parse HEAD` + docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} + outputs: + DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. if: (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) @@ -119,7 +124,11 @@ jobs: - name: Install dependencies run: make install-python-ci-dependencies - name: Test python - run: FEAST_USAGE=False IS_TEST=True pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5 + env: + FEAST_SERVER_DOCKER_IMAGE_TAG: ${{needs.build-docker-image.outputs.DOCKER_IMAGE_TAG}} + FEAST_USAGE: False + IS_TEST: True + run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5 - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index 1e88e60435..c2b750c423 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -37,3 +37,6 @@ # Default FTS port DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT = 6569 + +# Environment variable for feature server docker image tag +DOCKER_IMAGE_TAG_ENV_NAME: str = "FEAST_SERVER_DOCKER_IMAGE_TAG" \ No newline at end of file diff --git a/sdk/python/feast/infra/aws.py b/sdk/python/feast/infra/aws.py index 2ff38de9f9..f60ea5272a 100644 --- a/sdk/python/feast/infra/aws.py +++ b/sdk/python/feast/infra/aws.py @@ -18,6 +18,7 @@ AWS_LAMBDA_FEATURE_SERVER_REPOSITORY, FEAST_USAGE, FEATURE_STORE_YAML_ENV_NAME, + DOCKER_IMAGE_TAG_ENV_NAME, ) from feast.entity import Entity from feast.errors import ( @@ -322,27 +323,21 @@ def _get_lambda_name(project: str): def _get_docker_image_version() -> str: """Returns a version for the feature server Docker image. + If the feast.constants.DOCKER_IMAGE_TAG_ENV_NAME environment variable is set, + we return that (mostly used for integration tests, but can be used for local testing too). + For public Feast releases this equals to the Feast SDK version modified by replacing "." with "_". For example, Feast SDK version "0.14.1" would correspond to Docker image version "0_14_1". - For integration tests this equals to the git commit hash of HEAD. This is necessary, - because integration tests need to use images built from the same commit hash. - During development (when Feast is installed in editable mode) this equals to the Feast SDK version modified by removing the "dev..." suffix and replacing "." with "_". For example, Feast SDK version "0.14.1.dev41+g1cbfa225.d20211103" would correspond to Docker image version "0_14_1". This way, Feast SDK will use an already existing Docker image built during the previous public release. """ - if flags_helper.is_test(): - # Note: this should be in sync with https://github.com/feast-dev/feast/blob/6fbe01b6e9a444dc77ec3328a54376f4d9387664/.github/workflows/pr_integration_tests.yml#L41 - return ( - subprocess.check_output( - ["git", "rev-parse", "HEAD"], cwd=Path(__file__).resolve().parent - ) - .decode() - .strip() - ) + tag = os.environ.get(DOCKER_IMAGE_TAG_ENV_NAME) + if tag is not None: + return tag else: version = get_version() if "dev" in version: From 75bf9f83f65168f3f966a31bd63a81f0df669661 Mon Sep 17 00:00:00 2001 From: Tsotne Tabidze Date: Fri, 19 Nov 2021 15:50:25 -0800 Subject: [PATCH 2/2] Also fix integration_tests.yml Signed-off-by: Tsotne Tabidze --- .github/workflows/integration_tests.yml | 16 ++++++++++++---- .github/workflows/pr_integration_tests.yml | 1 - sdk/python/feast/constants.py | 2 +- sdk/python/feast/infra/aws.py | 4 +--- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 666bb08378..80cf794b09 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -23,17 +23,21 @@ jobs: - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 + - name: Set ECR image tag + id: image-tag + run: echo '::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`' - name: Build and push env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: feast-python-server - # Note: the image tags should be in sync with sdk/python/feast/infra/aws.py:_get_docker_image_version run: | docker build \ --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ - --tag $ECR_REGISTRY/$ECR_REPOSITORY:`git rev-parse HEAD` \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:`git rev-parse HEAD` + docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} + outputs: + DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: needs: build-docker-image runs-on: ${{ matrix.os }} @@ -99,7 +103,11 @@ jobs: - name: Install dependencies run: make install-python-ci-dependencies - name: Test python - run: FEAST_USAGE=False IS_TEST=True pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5 + env: + FEAST_SERVER_DOCKER_IMAGE_TAG: ${{needs.build-docker-image.outputs.DOCKER_IMAGE_TAG}} + FEAST_USAGE: False + IS_TEST: True + run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5 - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 31533fa29a..e3c5fd384c 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -41,7 +41,6 @@ jobs: env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: feast-python-server - # Note: the image tags should be in sync with sdk/python/feast/infra/aws.py:_get_docker_image_version run: | docker build \ --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index c2b750c423..e2e987dd77 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -39,4 +39,4 @@ DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT = 6569 # Environment variable for feature server docker image tag -DOCKER_IMAGE_TAG_ENV_NAME: str = "FEAST_SERVER_DOCKER_IMAGE_TAG" \ No newline at end of file +DOCKER_IMAGE_TAG_ENV_NAME: str = "FEAST_SERVER_DOCKER_IMAGE_TAG" diff --git a/sdk/python/feast/infra/aws.py b/sdk/python/feast/infra/aws.py index f60ea5272a..04b4abb48b 100644 --- a/sdk/python/feast/infra/aws.py +++ b/sdk/python/feast/infra/aws.py @@ -2,7 +2,6 @@ import hashlib import logging import os -import subprocess import uuid from datetime import datetime from pathlib import Path @@ -12,13 +11,12 @@ from colorama import Fore, Style -from feast import flags_helper from feast.constants import ( AWS_LAMBDA_FEATURE_SERVER_IMAGE, AWS_LAMBDA_FEATURE_SERVER_REPOSITORY, + DOCKER_IMAGE_TAG_ENV_NAME, FEAST_USAGE, FEATURE_STORE_YAML_ENV_NAME, - DOCKER_IMAGE_TAG_ENV_NAME, ) from feast.entity import Entity from feast.errors import (