From c9fb13c6612f0b418d3598e58f794610cc3520e6 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 11 Feb 2022 16:18:39 +0100 Subject: [PATCH] Prepare to switch to debian bullseye In order to switch smoothly to Debian Bullseye we need to run a few more tests and see if the performance impact we observe is consistent (the tests with Public Runners seems to be killed more frequently there). This requires some preparatory work - namely add the capability of overriding the DEBIAN_VERSION via PR - because otherwise Build Images workflow that runs from main will build buster images until the change is actually merged. This change introduce this capability without switching to Bullseye yet - it will unblock the actual PR that will make the change from failing in PRs though. --- .github/workflows/build-images.yml | 10 ++++++++-- Dockerfile | 3 +-- Dockerfile.ci | 3 +-- dev/retag_docker_images.py | 3 --- .../customizing/add-build-essential-custom.sh | 3 ++- .../docker-examples/customizing/custom-sources.sh | 3 ++- .../customizing/github-different-repository.sh | 4 +++- .../docker-examples/customizing/github-main.sh | 4 +++- .../docker-examples/customizing/github-v2-2-test.sh | 3 ++- .../customizing/pypi-dev-runtime-deps.sh | 3 ++- .../customizing/pypi-extras-and-deps.sh | 3 ++- .../restricted/restricted_environments.sh | 4 +++- .../docker-images-recipes/hadoop.Dockerfile | 3 +-- ...up_clusters_and_run_kubernetes_tests_in_parallel.sh | 1 + ...ade_cluster_with_different_executors_in_parallel.sh | 1 + scripts/ci/libraries/_build_images.sh | 2 +- scripts/ci/libraries/_docker_engine_resources.sh | 4 ++-- scripts/ci/libraries/_initialization.sh | 5 ++--- scripts/ci/testing/ci_run_airflow_testing.sh | 1 - 19 files changed, 37 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index f7bf0ff407401..3d77b5119599b 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -186,7 +186,7 @@ jobs: - name: "Retrieve DEFAULTS from the _initialization.sh" # We cannot "source" the script here because that would be a security problem (we cannot run # any code that comes from the sources coming from the PR. Therefore we extract the - # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH via custom grep/awk/sed commands + # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands id: defaults run: | DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \ @@ -196,6 +196,9 @@ jobs: scripts/ci/libraries/_initialization.sh | \ awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV + DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \ + awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') + echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV - name: > Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder to use ci/scripts from there. @@ -263,7 +266,7 @@ jobs: - name: "Retrieve DEFAULTS from the _initialization.sh" # We cannot "source" the script here because that would be a security problem (we cannot run # any code that comes from the sources coming from the PR. Therefore we extract the - # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH via custom grep/awk/sed commands + # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands id: defaults run: | DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \ @@ -273,6 +276,9 @@ jobs: scripts/ci/libraries/_initialization.sh | \ awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV + DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \ + cut -d "=" -f 3 | sed s'/["}]//g') + echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV - name: > Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder to use ci/scripts from there. diff --git a/Dockerfile b/Dockerfile index 54e127f9523eb..4f00a1d03c618 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,6 +68,7 @@ FROM ${PYTHON_BASE_IMAGE} as airflow-build-image SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] ARG PYTHON_BASE_IMAGE + ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \ DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 @@ -348,7 +349,6 @@ SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "n ARG AIRFLOW_UID LABEL org.apache.airflow.distro="debian" \ - org.apache.airflow.distro.version="buster" \ org.apache.airflow.module="airflow" \ org.apache.airflow.component="airflow" \ org.apache.airflow.image="airflow" \ @@ -512,7 +512,6 @@ ARG AIRFLOW_IMAGE_DATE_CREATED ENV BUILD_ID=${BUILD_ID} COMMIT_SHA=${COMMIT_SHA} LABEL org.apache.airflow.distro="debian" \ - org.apache.airflow.distro.version="buster" \ org.apache.airflow.module="airflow" \ org.apache.airflow.component="airflow" \ org.apache.airflow.image="airflow" \ diff --git a/Dockerfile.ci b/Dockerfile.ci index a9becabefa1a0..def0d56f86b3a 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -22,7 +22,7 @@ FROM ${PYTHON_BASE_IMAGE} as main # xtrace - to show commands executed) SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] -ARG PYTHON_BASE_IMAGE="python:3.7-slim-buster" +ARG PYTHON_BASE_IMAGE ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" # By increasing this number we can do force build of all dependencies @@ -373,7 +373,6 @@ RUN ln -sf /usr/bin/dumb-init /usr/local/bin/dumb-init EXPOSE 8080 LABEL org.apache.airflow.distro="debian" \ - org.apache.airflow.distro.version="buster" \ org.apache.airflow.module="airflow" \ org.apache.airflow.component="airflow" \ org.apache.airflow.image="airflow-ci" \ diff --git a/dev/retag_docker_images.py b/dev/retag_docker_images.py index 459f2c57c09e3..0bb3166621685 100755 --- a/dev/retag_docker_images.py +++ b/dev/retag_docker_images.py @@ -37,11 +37,8 @@ GHCR_IO_IMAGES = [ - "{prefix}/{repo}/{branch}/ci-manifest/python{python_version}:latest", "{prefix}/{repo}/{branch}/ci/python{python_version}:latest", - "{prefix}/{repo}/{branch}/prod-build/python{python_version}:latest", "{prefix}/{repo}/{branch}/prod/python{python_version}:latest", - "{prefix}/{repo}/{branch}/python:{python_version}-slim-buster", ] diff --git a/docs/docker-stack/docker-examples/customizing/add-build-essential-custom.sh b/docs/docker-stack/docker-examples/customizing/add-build-essential-custom.sh index 81ad1a8ea3886..743a0686f915c 100755 --- a/docs/docker-stack/docker-examples/customizing/add-build-essential-custom.sh +++ b/docs/docker-stack/docker-examples/customizing/add-build-essential-custom.sh @@ -23,10 +23,11 @@ cd "${AIRFLOW_SOURCES}" # [START build] export AIRFLOW_VERSION=2.2.2 +export DEBIAN_VERSION="buster" docker build . \ --pull \ - --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \ + --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-${DEBIAN_VERSION}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg ADDITIONAL_PYTHON_DEPS="mpi4py" \ --build-arg ADDITIONAL_DEV_APT_DEPS="libopenmpi-dev" \ diff --git a/docs/docker-stack/docker-examples/customizing/custom-sources.sh b/docs/docker-stack/docker-examples/customizing/custom-sources.sh index a7a65f2f10373..829545fbb11a1 100755 --- a/docs/docker-stack/docker-examples/customizing/custom-sources.sh +++ b/docs/docker-stack/docker-examples/customizing/custom-sources.sh @@ -23,10 +23,11 @@ cd "${AIRFLOW_SOURCES}" # [START build] export AIRFLOW_VERSION=2.2.2 +export DEBIAN_VERSION="buster" docker build . -f Dockerfile \ --pull \ - --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \ + --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-${DEBIAN_VERSION}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="slack,odbc" \ --build-arg ADDITIONAL_PYTHON_DEPS=" \ diff --git a/docs/docker-stack/docker-examples/customizing/github-different-repository.sh b/docs/docker-stack/docker-examples/customizing/github-different-repository.sh index 5a0a1798db8ca..93c590c66038c 100755 --- a/docs/docker-stack/docker-examples/customizing/github-different-repository.sh +++ b/docs/docker-stack/docker-examples/customizing/github-different-repository.sh @@ -21,9 +21,11 @@ set -euo pipefail AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)" cd "${AIRFLOW_SOURCES}" # [START build] +export DEBIAN_VERSION="buster" + docker build . \ --pull \ - --build-arg PYTHON_BASE_IMAGE="python:3.8-slim-buster" \ + --build-arg PYTHON_BASE_IMAGE="python:3.8-slim-${DEBIAN_VERSION}" \ --build-arg AIRFLOW_INSTALLATION_METHOD="https://github.com/potiuk/airflow/archive/main.tar.gz#egg=apache-airflow" \ --build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-main" \ --build-arg CONSTRAINTS_GITHUB_REPOSITORY="potiuk/airflow" \ diff --git a/docs/docker-stack/docker-examples/customizing/github-main.sh b/docs/docker-stack/docker-examples/customizing/github-main.sh index fc1f514beaf7c..1bdf32a3dbcea 100755 --- a/docs/docker-stack/docker-examples/customizing/github-main.sh +++ b/docs/docker-stack/docker-examples/customizing/github-main.sh @@ -22,9 +22,11 @@ AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)" cd "${AIRFLOW_SOURCES}" # [START build] +export DEBIAN_VERSION="buster" + docker build . \ --pull \ - --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \ + --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-${DEBIAN_VERSION}" \ --build-arg AIRFLOW_INSTALLATION_METHOD="https://github.com/apache/airflow/archive/main.tar.gz#egg=apache-airflow" \ --build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-main" \ --tag "my-github-main:0.0.1" diff --git a/docs/docker-stack/docker-examples/customizing/github-v2-2-test.sh b/docs/docker-stack/docker-examples/customizing/github-v2-2-test.sh index 16c4b20ac8307..10514506f1314 100755 --- a/docs/docker-stack/docker-examples/customizing/github-v2-2-test.sh +++ b/docs/docker-stack/docker-examples/customizing/github-v2-2-test.sh @@ -22,9 +22,10 @@ AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)" cd "${AIRFLOW_SOURCES}" # [START build] +export DEBIAN_VERSION="buster" docker build . \ --pull \ - --build-arg PYTHON_BASE_IMAGE="python:3.8-slim-buster" \ + --build-arg PYTHON_BASE_IMAGE="python:3.8-slim-${DEBIAN_VERSION}" \ --build-arg AIRFLOW_INSTALLATION_METHOD="https://github.com/apache/airflow/archive/v2-2-test.tar.gz#egg=apache-airflow" \ --build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-2-2" \ --tag "my-github-v2-2:0.0.1" diff --git a/docs/docker-stack/docker-examples/customizing/pypi-dev-runtime-deps.sh b/docs/docker-stack/docker-examples/customizing/pypi-dev-runtime-deps.sh index d8f9de08840e5..06ac690de5bd4 100755 --- a/docs/docker-stack/docker-examples/customizing/pypi-dev-runtime-deps.sh +++ b/docs/docker-stack/docker-examples/customizing/pypi-dev-runtime-deps.sh @@ -23,10 +23,11 @@ cd "${AIRFLOW_SOURCES}" # [START build] export AIRFLOW_VERSION=2.2.2 +export DEBIAN_VERSION="buster" docker build . \ --pull \ - --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \ + --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-${DEBIAN_VERSION}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="jdbc" \ --build-arg ADDITIONAL_PYTHON_DEPS="pandas" \ diff --git a/docs/docker-stack/docker-examples/customizing/pypi-extras-and-deps.sh b/docs/docker-stack/docker-examples/customizing/pypi-extras-and-deps.sh index cd96e5bf60798..3e9a1a170d0a7 100755 --- a/docs/docker-stack/docker-examples/customizing/pypi-extras-and-deps.sh +++ b/docs/docker-stack/docker-examples/customizing/pypi-extras-and-deps.sh @@ -23,10 +23,11 @@ cd "${AIRFLOW_SOURCES}" # [START build] export AIRFLOW_VERSION=2.2.2 +export DEBIAN_VERSION="buster" docker build . \ --pull \ - --build-arg PYTHON_BASE_IMAGE="python:3.8-slim-buster" \ + --build-arg PYTHON_BASE_IMAGE="python:3.8-slim-${DEBIAN_VERSION}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="mssql,hdfs" \ --build-arg ADDITIONAL_PYTHON_DEPS="oauth2client" \ diff --git a/docs/docker-stack/docker-examples/restricted/restricted_environments.sh b/docs/docker-stack/docker-examples/restricted/restricted_environments.sh index 205447161e274..b287218f45610 100755 --- a/docs/docker-stack/docker-examples/restricted/restricted_environments.sh +++ b/docs/docker-stack/docker-examples/restricted/restricted_environments.sh @@ -34,9 +34,11 @@ pip download --dest docker-context-files \ # [END download] # [START build] +export DEBIAN_VERSION="buster" + docker build . \ --pull \ - --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \ + --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-${DEBIAN_VERSION}" \ --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg INSTALL_MYSQL_CLIENT="false" \ diff --git a/docs/docker-stack/docker-images-recipes/hadoop.Dockerfile b/docs/docker-stack/docker-images-recipes/hadoop.Dockerfile index 5ab1cdce4efc2..edb321d4138db 100644 --- a/docs/docker-stack/docker-images-recipes/hadoop.Dockerfile +++ b/docs/docker-stack/docker-images-recipes/hadoop.Dockerfile @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. ARG BASE_AIRFLOW_IMAGE - FROM ${BASE_AIRFLOW_IMAGE} SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] @@ -24,7 +23,7 @@ USER 0 RUN mkdir -pv /usr/share/man/man1 \ && mkdir -pv /usr/share/man/man7 \ && curl -fsSL https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - \ - && echo 'deb https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ buster main' > \ + && echo "deb https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ $(lsb_release -cs) main" > \ /etc/apt/sources.list.d/adoptopenjdk.list \ && apt-get update \ && apt-get install --no-install-recommends -y \ diff --git a/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh b/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh index d42aaec9c595a..2754f927dc443 100755 --- a/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh +++ b/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh @@ -22,6 +22,7 @@ source "${LIBRARIES_DIR}/_all_libs.sh" export SEMAPHORE_NAME="kubernetes-tests" initialization::set_output_color_variables +initialization::initialize_image_build_variables parallel::make_sure_gnu_parallel_is_installed parallel::make_sure_python_versions_are_specified diff --git a/scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh b/scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh index b878c70886aae..797312f032007 100755 --- a/scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh +++ b/scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh @@ -22,6 +22,7 @@ source "${LIBRARIES_DIR}/_all_libs.sh" export SEMAPHORE_NAME="kubernetes-tests-upgrade" initialization::set_output_color_variables +initialization::initialize_image_build_variables parallel::make_sure_gnu_parallel_is_installed parallel::make_sure_python_versions_are_specified diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index a50a6c3fc1797..55dab26b3a3b6 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -263,7 +263,7 @@ function build_images::get_github_container_registry_image_prefix() { function build_images::get_docker_cache_image_names() { # Python base image to use - export PYTHON_BASE_IMAGE="python:${PYTHON_MAJOR_MINOR_VERSION}-slim-buster" + export PYTHON_BASE_IMAGE="python:${PYTHON_MAJOR_MINOR_VERSION}-slim-${DEBIAN_VERSION}" local image_name image_name="ghcr.io/$(build_images::get_github_container_registry_image_prefix)" diff --git a/scripts/ci/libraries/_docker_engine_resources.sh b/scripts/ci/libraries/_docker_engine_resources.sh index af836815f98cf..9bb08c404d8b9 100644 --- a/scripts/ci/libraries/_docker_engine_resources.sh +++ b/scripts/ci/libraries/_docker_engine_resources.sh @@ -33,12 +33,12 @@ function docker_engine_resources::print_overall_stats() { } function docker_engine_resources::get_available_cpus_in_docker() { - CPUS_AVAILABLE_FOR_DOCKER=$(docker run --rm "debian:buster-slim" grep -cE 'cpu[0-9]+' /proc/stat) + CPUS_AVAILABLE_FOR_DOCKER=$(docker run --rm "debian:${DEBIAN_VERSION}-slim" grep -cE 'cpu[0-9]+' /proc/stat) export CPUS_AVAILABLE_FOR_DOCKER } function docker_engine_resources::get_available_memory_in_docker() { - MEMORY_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash "debian:buster-slim" -c 'echo $(($(getconf _PHYS_PAGES) * $(getconf PAGE_SIZE) / (1024 * 1024)))') + MEMORY_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash "debian:${DEBIAN_VERSION}-slim" -c 'echo $(($(getconf _PHYS_PAGES) * $(getconf PAGE_SIZE) / (1024 * 1024)))') export MEMORY_AVAILABLE_FOR_DOCKER } diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index e860d19e00bbb..d6638b49fe8ba 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -352,6 +352,8 @@ function initialization::initialize_image_build_variables() { # Default build id export CI_BUILD_ID="${CI_BUILD_ID:="0"}" + export DEBIAN_VERSION=${DEBIAN_VERSION:="buster"} + # Default extras used for building Production image. The canonical source of this information is in the Dockerfile DEFAULT_PROD_EXTRAS=$(grep "ARG AIRFLOW_EXTRAS=" "${AIRFLOW_SOURCES}/Dockerfile" | awk 'BEGIN { FS="=" } { print $2 }' | tr -d '"') @@ -632,10 +634,7 @@ function initialization::initialize_common_environment() { function initialization::set_default_python_version_if_empty() { # default version of python used to tag the "main" and "latest" images in DockerHub export DEFAULT_PYTHON_MAJOR_MINOR_VERSION=3.7 - - # default python Major/Minor version export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}} - } function initialization::summarize_build_environment() { diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh index e3f93495acbe0..2d4ed89f8315b 100755 --- a/scripts/ci/testing/ci_run_airflow_testing.sh +++ b/scripts/ci/testing/ci_run_airflow_testing.sh @@ -135,7 +135,6 @@ function run_all_test_types_in_parallel() { parallel::print_job_summary_and_return_status_code } - testing::skip_tests_if_requested build_images::prepare_ci_build