From 9957f18cc1be9a2882c6f5c1c65407eff800df16 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 27 May 2020 17:09:11 +0200 Subject: [PATCH] Additional python extras and deps can be set in breeze (#9035) Closes #8604 Closes #8866 (cherry picked from commit 738667082d32d3ef93ec2cd6c3735ff3691ba1cc) --- BREEZE.rst | 12 ++++++++++++ Dockerfile | 4 ++-- Dockerfile.ci | 3 ++- breeze | 14 ++++++++++++++ breeze-complete | 1 + scripts/ci/_utils.sh | 12 +++++++++++- 6 files changed, 42 insertions(+), 4 deletions(-) diff --git a/BREEZE.rst b/BREEZE.rst index 44af4e5c49bd7..262ddcd2d1f8d 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -774,6 +774,12 @@ This is the current syntax for `./breeze <./breeze>`_: async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack, ssh,statsd,virtualenv + --additional-extras + Additional extras to pass to build images The default is no additional extras. + + --additional-python-deps + Additional python dependencies to use when building the images. + -C, --force-clean-images Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. @@ -1312,6 +1318,12 @@ This is the current syntax for `./breeze <./breeze>`_: async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack, ssh,statsd,virtualenv + --additional-extras + Additional extras to pass to build images The default is no additional extras. + + --additional-python-deps + Additional python dependencies to use when building the images. + -C, --force-clean-images Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. diff --git a/Dockerfile b/Dockerfile index ef3a9887d0c89..ac27823391f2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -169,10 +169,10 @@ ARG AIRFLOW_VERSION ENV AIRFLOW_VERSION=${AIRFLOW_VERSION} ARG AIRFLOW_EXTRAS -ARG ADDITIONAL_AIRFLOW_EXTRAS +ARG ADDITIONAL_AIRFLOW_EXTRAS="" ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} -ARG ADDITIONAL_PYTHON_DEPS +ARG ADDITIONAL_PYTHON_DEPS="" ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} ARG AIRFLOW_INSTALL_SOURCES="." diff --git a/Dockerfile.ci b/Dockerfile.ci index 15f373268d8f6..2d107aa6aa764 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -295,7 +295,8 @@ ENV AIRFLOW_BRANCH=${AIRFLOW_BRANCH} # Airflow Extras installed ARG AIRFLOW_EXTRAS="all" -ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS} +ARG ADDITIONAL_AIRFLOW_EXTRAS="" +ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} RUN echo "Installing with extras: ${AIRFLOW_EXTRAS}." diff --git a/breeze b/breeze index 7f5958b4d9332..b2e3c67548b30 100755 --- a/breeze +++ b/breeze @@ -772,6 +772,14 @@ function parse_arguments() { export AIRFLOW_EXTRAS="${2}" echo "Extras : ${AIRFLOW_EXTRAS}" shift 2 ;; + --additional-extras) + export ADDITIONAL_AIRFLOW_EXTRAS="${2}" + echo "Additional extras : ${ADDITIONAL_AIRFLOW_EXTRAS}" + shift 2 ;; + --additional-python-deps) + export ADDITIONAL_PYTHON_DEPS="${2}" + echo "Additional python dependencies: ${ADDITIONAL_PYTHON_DEPS}" + shift 2 ;; -D|--dockerhub-user) export DOCKERHUB_USER="${2}" echo "Dockerhub user ${DOCKERHUB_USER}" @@ -1527,6 +1535,12 @@ ${FORMATTED_DEFAULT_CI_EXTRAS} Production image: ${FORMATTED_DEFAULT_PROD_EXTRAS} +--additional-extras + Additional extras to pass to build images The default is no additional extras. + +--additional-python-deps + Additional python dependencies to use when building the images. + -C, --force-clean-images Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. diff --git a/breeze-complete b/breeze-complete index 31648514e7317..8afe550f68cc2 100644 --- a/breeze-complete +++ b/breeze-complete @@ -99,6 +99,7 @@ verbose assume-yes assume-no assume-quit forward-credentials force-build-images force-pull-images production-image extras: force-clean-images use-local-cache dockerhub-user: dockerhub-repo: push-images postgres-version: mysql-version: +additional-extras: additional-python-deps: " export BREEZE_COMMANDS=" diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh index 7bbd2dd24379b..85aeb91db8d79 100644 --- a/scripts/ci/_utils.sh +++ b/scripts/ci/_utils.sh @@ -1311,6 +1311,8 @@ Docker building ${AIRFLOW_CI_IMAGE}. --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_BRANCH="${BRANCH_NAME}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ + --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \ + --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \ --build-arg AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}" \ --build-arg UPGRADE_TO_LATEST_REQUIREMENTS="${UPGRADE_TO_LATEST_REQUIREMENTS}" \ "${DOCKER_CACHE_CI_DIRECTIVE[@]}" \ @@ -1364,6 +1366,8 @@ function build_prod_image() { --build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ + --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \ + --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \ "${DOCKER_CACHE_PROD_BUILD_DIRECTIVE[@]}" \ -t "${AIRFLOW_PROD_BUILD_IMAGE}" \ --target "airflow-build-image" \ @@ -1372,6 +1376,8 @@ function build_prod_image() { "${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \ --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \ --build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \ + --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \ + --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ "${DOCKER_CACHE_PROD_DIRECTIVE[@]}" \ @@ -1473,8 +1479,9 @@ function prepare_ci_build() { export IMAGE_DESCRIPTION="Airflow CI" export AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="true" export AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS:="${DEFAULT_CI_EXTRAS}"}" + export ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS:=""}" + export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}" export AIRFLOW_IMAGE="${AIRFLOW_CI_IMAGE}" - go_to_airflow_sources fix_group_permissions } @@ -1532,7 +1539,10 @@ function prepare_prod_build() { export THE_IMAGE_TYPE="PROD" export IMAGE_DESCRIPTION="Airflow production" export AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS:="${DEFAULT_PROD_EXTRAS}"}" + export ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS:=""}" + export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}" export AIRFLOW_IMAGE="${AIRFLOW_PROD_IMAGE}" + if [[ ${CACHE_REGISTRY=""} != "" ]]; then echo "${CACHE_REGISTRY_PASSWORD}" | docker login \ --username "${CACHE_REGISTRY_USERNAME}" \