From c4ec9c19695bef64c4cc9ab7527b7b0c4c4e0cfb Mon Sep 17 00:00:00 2001 From: Elizabeth Thompson Date: Tue, 12 Sep 2023 17:44:29 -0700 Subject: [PATCH] add latest-official docker tag --- .github/workflows/docker-release.yml | 3 +- .github/workflows/docker.yml | 2 +- .../docker_build_push.sh | 33 +++++ scripts/tag_latest_release.sh | 140 +++++++++++++----- tests/unit_tests/fixtures/bash_mock.py | 44 ++++++ .../scripts/docker_build_push_test.py | 44 ++++++ .../scripts/tag_latest_release_test.py | 49 ++++++ 7 files changed, 274 insertions(+), 41 deletions(-) rename {.github/workflows => scripts}/docker_build_push.sh (84%) create mode 100644 tests/unit_tests/fixtures/bash_mock.py create mode 100644 tests/unit_tests/scripts/docker_build_push_test.py create mode 100644 tests/unit_tests/scripts/tag_latest_release_test.py diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index b079206f51110..7cfba73299b43 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -38,4 +38,5 @@ jobs: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} run: | - .github/workflows/docker_build_push.sh + GITHUB_RELEASE_TAG_NAME="${{ github.event.release.tag_name }}" + ./scripts/docker_build_push.sh "$GITHUB_RELEASE_TAG_NAME" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 983267ff602f7..6160d3cc1f595 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -44,7 +44,7 @@ jobs: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} run: | - .github/workflows/docker_build_push.sh + ./scripts/docker_build_push.sh - name: Build ephemeral env image if: github.event_name == 'pull_request' diff --git a/.github/workflows/docker_build_push.sh b/scripts/docker_build_push.sh similarity index 84% rename from .github/workflows/docker_build_push.sh rename to scripts/docker_build_push.sh index 3e90af45f1ada..17d8602e06025 100755 --- a/.github/workflows/docker_build_push.sh +++ b/scripts/docker_build_push.sh @@ -17,6 +17,8 @@ # set -eo pipefail +GITHUB_RELEASE_TAG_NAME="$1" + SHA=$(git rev-parse HEAD) REPO_NAME="apache/superset" @@ -32,10 +34,26 @@ else LATEST_TAG="${REFSPEC}" fi + if [[ "${REFSPEC}" == "master" ]]; then LATEST_TAG="latest" fi +# get the latest release tag +if [ -n "${GITHUB_RELEASE_TAG_NAME}" ]; then + output=$(source ./scripts/tag_latest_release.sh "${GITHUB_RELEASE_TAG_NAME}" --dry-run) || true + SKIP_TAG=$(echo "${output}" | grep "SKIP_TAG" | cut -d'=' -f2) + if [[ "${SKIP_TAG}" == "SKIP_TAG::false" ]]; then + LATEST_TAG="latest-official" + fi +fi + +if [[ "${TEST_ENV}" == "true" ]]; then + # don't run the build in test environment + exit 0 +fi + + cat<" + echo "::set-output name=SKIP_TAG::true" exit 1 fi -if [ -z "$(git show-ref ${GITHUB_TAG_NAME})" ]; then +if [ -z "$(git_show_ref)" ]; then echo "The tag ${GITHUB_TAG_NAME} does not exist. Please use a different tag." - exit 1 + echo "::set-output name=SKIP_TAG::true" + exit 0 fi # check that this tag only contains a proper semantic version @@ -63,36 +113,41 @@ if ! [[ ${GITHUB_TAG_NAME} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] then echo "This tag ${GITHUB_TAG_NAME} is not a valid release version. Not tagging." echo "::set-output name=SKIP_TAG::true" - exit 0 + exit 1 fi ## split the current GITHUB_TAG_NAME into an array at the dot -IFS=$'.' -THIS_TAG_NAME=(${GITHUB_TAG_NAME}) || echo 'not found' +THIS_TAG_NAME=$(split_string "${GITHUB_TAG_NAME}" ".") # look up the 'latest' tag on git -LATEST_TAG_LIST=$(git show-ref latest && git show --pretty=tformat:%d -s latest | grep tag:) || echo 'not found' +LATEST_TAG_LIST=$(get_latest_tag_list) || echo 'not found' # if 'latest' tag doesn't exist, then set this commit to latest if [[ -z "$LATEST_TAG_LIST" ]] then - # move on to next task echo "there are no latest tags yet, so I'm going to start by tagging this sha as the latest" run_git_tag + exit 0 fi -## get all tags that use the same sha as the latest tag. split at comma. -IFS=$',' -LATEST_TAGS=($LATEST_TAG_LIST) +# remove parenthesis and tag: from the list of tags +LATEST_TAGS_STRINGS=$(echo "$LATEST_TAG_LIST" | sed 's/tag: \([^,]*\)/\1/g' | tr -d '()') -## loop over those tags and only take action on the one that isn't tagged 'latest' -## that one will have the version number tag -for (( i=0; i<${#LATEST_TAGS[@]}; i++ )) +LATEST_TAGS=$(split_string "$LATEST_TAGS_STRINGS" ",") +TAGS=($(split_string "$LATEST_TAGS" " ")) + +# Initialize a flag for comparison result +compare_result="" + +# Iterate through the tags of the latest release +for tag in $TAGS do - if [[ ${LATEST_TAGS[$i]} != *"latest"* ]] - then + if [[ $tag == "latest" ]]; then + continue + else ## extract just the version from this tag - LATEST_RELEASE_TAG=$(echo "${LATEST_TAGS[$i]}" | sed -E -e 's/tag:|\(|\)|[[:space:]]*//g') + LATEST_RELEASE_TAG="$tag" + echo "LATEST_RELEASE_TAG: ${LATEST_RELEASE_TAG}" # check that this only contains a proper semantic version if ! [[ ${LATEST_RELEASE_TAG} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] @@ -101,28 +156,35 @@ do continue fi echo "The current release with the latest tag is version ${LATEST_RELEASE_TAG}" - - ## remove the sha from the latest tag and split into an array- split at the dot - IFS=$'.' - LATEST_RELEASE_TAG_SPLIT=(${LATEST_RELEASE_TAG}) - - for (( j=0; j<${#THIS_TAG_NAME[@]}; j++ )) - do - ## if this value is greater than the latest release, then tag it, if it's lower, then stop, if it's - ## the same then move on to the next index - if [[ ${THIS_TAG_NAME[$j]} -gt ${LATEST_RELEASE_TAG_SPLIT[$j]} ]] - then - echo "This release tag ${GITHUB_TAG_NAME} is the latest. Tagging it" - run_git_tag - - elif [[ ${THIS_TAG_NAME[$j]} -lt ${LATEST_RELEASE_TAG_SPLIT[$j]} ]] - then - continue - fi + # Split the version strings into arrays + THIS_TAG_NAME_ARRAY=($(split_string "$THIS_TAG_NAME" ".")) + LATEST_RELEASE_TAG_ARRAY=($(split_string "$LATEST_RELEASE_TAG" ".")) + + # Iterate through the components of the version strings + for (( j=0; j<${#THIS_TAG_NAME_ARRAY[@]}; j++ )); do + echo "Comparing ${THIS_TAG_NAME_ARRAY[$j]} to ${LATEST_RELEASE_TAG_ARRAY[$j]}" + if [[ $((THIS_TAG_NAME_ARRAY[$j])) > $((LATEST_RELEASE_TAG_ARRAY[$j])) ]]; then + compare_result="greater" + break + elif [[ $((THIS_TAG_NAME_ARRAY[$j])) < $((LATEST_RELEASE_TAG_ARRAY[$j])) ]]; then + compare_result="lesser" + break + fi done fi done -echo "This release tag ${GITHUB_TAG_NAME} is not the latest. Not tagging." -# if you've gotten this far, then we don't want to run any tags in the next step -echo "::set-output name=SKIP_TAG::true" +# Determine the result based on the comparison +if [[ -z "$compare_result" ]]; then + echo "Versions are equal" + echo "::set-output name=SKIP_TAG::true" +elif [[ "$compare_result" == "greater" ]]; then + echo "This release tag ${GITHUB_TAG_NAME} is newer than the latest." + echo "::set-output name=SKIP_TAG::false" + # Add other actions you want to perform for a newer version +elif [[ "$compare_result" == "lesser" ]]; then + echo "This release tag ${GITHUB_TAG_NAME} is older than the latest." + echo "This release tag ${GITHUB_TAG_NAME} is not the latest. Not tagging." + # if you've gotten this far, then we don't want to run any tags in the next step + echo "::set-output name=SKIP_TAG::true" +fi diff --git a/tests/unit_tests/fixtures/bash_mock.py b/tests/unit_tests/fixtures/bash_mock.py new file mode 100644 index 0000000000000..8ab8838b8820a --- /dev/null +++ b/tests/unit_tests/fixtures/bash_mock.py @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import subprocess + + +class BashMock: + @staticmethod + def tag_latest_release(tag): + bash_command = f"./scripts/tag_latest_release.sh {tag} --dry-run" + result = subprocess.run( + bash_command, + shell=True, + capture_output=True, + text=True, + env={"TEST_ENV": "true"}, + ) + return result + + @staticmethod + def docker_build_push(tag): + bash_command = f"./scripts/docker_build_push.sh {tag}" + result = subprocess.run( + bash_command, + shell=True, + capture_output=True, + text=True, + env={"TEST_ENV": "true", "GITHUB_REF": "refs/heads/master"}, + ) + return result diff --git a/tests/unit_tests/scripts/docker_build_push_test.py b/tests/unit_tests/scripts/docker_build_push_test.py new file mode 100644 index 0000000000000..8c37c461a428d --- /dev/null +++ b/tests/unit_tests/scripts/docker_build_push_test.py @@ -0,0 +1,44 @@ +import re +import subprocess +from unittest import mock +from unittest.mock import patch + +import pytest + +from tests.unit_tests.fixtures.bash_mock import BashMock + +original_run = subprocess.run + + +def wrapped(*args, **kwargs): + return original_run(*args, **kwargs) + + +@pytest.mark.parametrize( + "tag, expected_output", + [ + ("1.0.0", "LATEST_TAG is latest"), + ("2.1.0", "LATEST_TAG is latest"), + ("2.1.1", "LATEST_TAG is latest-official"), + ("3.0.0", "LATEST_TAG is latest-official"), + ("2.1.0rc1", "LATEST_TAG is latest"), + ("", "LATEST_TAG is latest"), + ("2.1", "LATEST_TAG is latest"), + ("does_not_exist", "LATEST_TAG is latest"), + ], +) +def test_tag_latest_release(tag, expected_output): + with mock.patch( + "tests.unit_tests.fixtures.bash_mock.subprocess.run", wraps=wrapped + ) as subprocess_mock: + result = BashMock.docker_build_push(tag) + + subprocess_mock.assert_called_once_with( + f"./scripts/docker_build_push.sh {tag}", + shell=True, + capture_output=True, + text=True, + env={"TEST_ENV": "true", "GITHUB_REF": "refs/heads/master"}, + ) + + assert re.search(expected_output, result.stdout, re.MULTILINE) diff --git a/tests/unit_tests/scripts/tag_latest_release_test.py b/tests/unit_tests/scripts/tag_latest_release_test.py new file mode 100644 index 0000000000000..7b15a1670a7b0 --- /dev/null +++ b/tests/unit_tests/scripts/tag_latest_release_test.py @@ -0,0 +1,49 @@ +import subprocess +from unittest import mock +from unittest.mock import patch + +import pytest + +from tests.unit_tests.fixtures.bash_mock import BashMock + +original_run = subprocess.run + + +def wrapped(*args, **kwargs): + return original_run(*args, **kwargs) + + +@pytest.mark.parametrize( + "tag, expected_output", + [ + ("1.0.0", "This release tag 1.0.0 is older than the latest."), + ("2.1.0", "Versions are equal\n::set-output name=SKIP_TAG::true"), + ("2.1.1", "This release tag 2.1.1 is newer than the latest."), + ("3.0.0", "This release tag 3.0.0 is newer than the latest."), + ("2.1.0rc1", "This tag 2.1.0rc1 is not a valid release version. Not tagging."), + ( + "", + "Missing tag parameter, usage: ./scripts/tag_latest_release.sh ", + ), + ("2.1", "This tag 2.1 is not a valid release version. Not tagging."), + ( + "does_not_exist", + "The tag does_not_exist does not exist. Please use a different tag.\n::set-output name=SKIP_TAG::true", + ), + ], +) +def test_tag_latest_release(tag, expected_output): + with mock.patch( + "tests.unit_tests.fixtures.bash_mock.subprocess.run", wraps=wrapped + ) as subprocess_mock: + result = BashMock.tag_latest_release(tag) + + subprocess_mock.assert_called_once_with( + f"./scripts/tag_latest_release.sh {tag} --dry-run", + shell=True, + capture_output=True, + text=True, + env={"TEST_ENV": "true"}, + ) + + assert expected_output in result.stdout