From c6e6d6dedd793b76e69ff2b7261bc7dc084d6cc7 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 25 Aug 2020 17:01:39 +0200 Subject: [PATCH] Helm Docker image sources are now included in the Airlfow codebase (#9650) We can now build all the images from Airlfow sources in a reproducible fashion and our users can use the helm chart based on the images build from official images + code in Airflow Codebase. We also have consistent versioning scheme based on calver version of releasing the images coupled with the version of the original package. Part of #9401 --- chart/dockerfiles/README.md | 28 ++++++ .../dockerfiles/pgbouncer-exporter/README.md | 21 +++++ chart/dockerfiles/pgbouncer/Dockerfile | 65 ++++++++++++++ chart/dockerfiles/pgbouncer/build_and_push.sh | 36 ++++++++ chart/dockerfiles/statsd-exporter/Dockerfile | 32 +++++++ .../statsd-exporter/build_and_push.sh | 36 ++++++++ .../dockerfiles/statsd-exporter/mappings.yml | 87 +++++++++++++++++++ chart/values.yaml | 12 +-- .../kubernetes/ci_deploy_app_to_kubernetes.sh | 1 - 9 files changed, 311 insertions(+), 7 deletions(-) create mode 100644 chart/dockerfiles/README.md create mode 100644 chart/dockerfiles/pgbouncer-exporter/README.md create mode 100644 chart/dockerfiles/pgbouncer/Dockerfile create mode 100755 chart/dockerfiles/pgbouncer/build_and_push.sh create mode 100644 chart/dockerfiles/statsd-exporter/Dockerfile create mode 100755 chart/dockerfiles/statsd-exporter/build_and_push.sh create mode 100644 chart/dockerfiles/statsd-exporter/mappings.yml diff --git a/chart/dockerfiles/README.md b/chart/dockerfiles/README.md new file mode 100644 index 0000000000000..8e663a082e039 --- /dev/null +++ b/chart/dockerfiles/README.md @@ -0,0 +1,28 @@ + + +Those are images that are needed for the Helm Chart. + +In each of the images you can find "build_and_push.sh" script that builds and pushes the image. + +You need to be a PMC with direct push access to "apache/airflow" DockerHub registry +to be able to push to the Airflow DockerHub registry. + +You can set the DOCKERHUB_USER variable to push to your own DockerHub user if you want + to test the image or build your own image. diff --git a/chart/dockerfiles/pgbouncer-exporter/README.md b/chart/dockerfiles/pgbouncer-exporter/README.md new file mode 100644 index 0000000000000..03614e00b4686 --- /dev/null +++ b/chart/dockerfiles/pgbouncer-exporter/README.md @@ -0,0 +1,21 @@ + + +The dockerfile and build script for pgbouncer-exporter can be found +in https://github.com/apache/airflow-pgbouncer-exporter diff --git a/chart/dockerfiles/pgbouncer/Dockerfile b/chart/dockerfiles/pgbouncer/Dockerfile new file mode 100644 index 0000000000000..584e4a343a33f --- /dev/null +++ b/chart/dockerfiles/pgbouncer/Dockerfile @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +ARG ALPINE_VERSION="3.12" + +FROM alpine:${ALPINE_VERSION} AS builder + +ARG PGBOUNCER_VERSION +ARG AIRFLOW_PGBOUNCER_VERSION + +ARG PGBOUNCER_SHA256="a0c13d10148f557e36ff7ed31793abb7a49e1f8b09aa2d4695d1c28fa101fee7" + +RUN apk --no-cache add make pkgconfig build-base libtool wget gcc g++ libevent-dev libressl-dev c-ares-dev ca-certificates +RUN wget https://github.com/pgbouncer/pgbouncer/releases/download/pgbouncer_$(echo ${PGBOUNCER_VERSION} | sed s/\\./_/g)/pgbouncer-${PGBOUNCER_VERSION}.tar.gz +RUN echo "${PGBOUNCER_SHA256} pgbouncer-${PGBOUNCER_VERSION}.tar.gz" | sha256sum -c - +RUN tar -xzvf pgbouncer-$PGBOUNCER_VERSION.tar.gz +WORKDIR pgbouncer-$PGBOUNCER_VERSION +RUN ./configure --prefix=/usr --disable-debug && make && make install +RUN mkdir /etc/pgbouncer +RUN cp ./etc/pgbouncer.ini /etc/pgbouncer/ +RUN touch /etc/pgbouncer/userlist.txt +RUN sed -i -e "s|logfile = |#logfile = |" \ + -e "s|pidfile = |#pidfile = |" \ + -e "s|listen_addr = .*|listen_addr = 0.0.0.0|" \ + -e "s|auth_type = .*|auth_type = md5|" \ + /etc/pgbouncer/pgbouncer.ini + +FROM alpine:${ALPINE_VERSION} + +ARG PGBOUNCER_VERSION +ARG AIRFLOW_PGBOUNCER_VERSION +ARG COMMIT_SHA + +LABEL org.apache.airflow.component="pgbouncer" +LABEL org.apache.airflow.pgbouncer.version="${PGBOUNCER_VERSION}" +LABEL org.apache.airflow.airflow_pgbouncer.version="${AIRFLOW_PGBOUNCER_VERSION}" +LABEL org.apache.airflow.commit_sha="${COMMIT_SHA}" + +MAINTAINER "Apache Airflow Community " + +RUN apk --no-cache add libevent libressl c-ares + +COPY --from=builder /etc/pgbouncer /etc/pgbouncer +COPY --from=builder /usr/bin/pgbouncer /usr/bin/pgbouncer + +# Healthcheck +HEALTHCHECK --interval=10s --timeout=3s CMD stat /tmp/.s.PGSQL.* + +EXPOSE 6432 + +# pgbouncer can't run as root, so let's drop to 'nobody' +ENTRYPOINT ["/usr/bin/pgbouncer", "-u", "nobody", "/etc/pgbouncer/pgbouncer.ini" ] diff --git a/chart/dockerfiles/pgbouncer/build_and_push.sh b/chart/dockerfiles/pgbouncer/build_and_push.sh new file mode 100755 index 0000000000000..a7dc7621dc5cf --- /dev/null +++ b/chart/dockerfiles/pgbouncer/build_and_push.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +set -euo pipefail +DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} +DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"} +PGBOUNCER_VERSION="1.14.0" +AIRFLOW_PGBOUNCER_VERSION="2020.07.10" +COMMIT_SHA=$(git rev-parse HEAD) + +cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1 + +TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:airflow-pgbouncer-${AIRFLOW_PGBOUNCER_VERSION}-${PGBOUNCER_VERSION}" + +docker build . \ + --pull \ + --build-arg "PGBOUNCER_VERSION=${PGBOUNCER_VERSION}" \ + --build-arg "AIRFLOW_PGBOUNCER_VERSION=${AIRFLOW_PGBOUNCER_VERSION}"\ + --build-arg "COMMIT_SHA=${COMMIT_SHA}" \ + --tag "${TAG}" + +docker push "${TAG}" diff --git a/chart/dockerfiles/statsd-exporter/Dockerfile b/chart/dockerfiles/statsd-exporter/Dockerfile new file mode 100644 index 0000000000000..4f61fc1017cfd --- /dev/null +++ b/chart/dockerfiles/statsd-exporter/Dockerfile @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +ARG STATSD_VERSION="missing_version" + +FROM prom/statsd-exporter:${STATSD_VERSION} + +ARG STATSD_VERSION +ARG AIRFLOW_STATSD_EXPORTER_VERSION +ARG COMMIT_SHA + +LABEL org.apache.airflow.component="statsd-exporter" +LABEL org.apache.airflow.stasd.version="${STATSD_VERSION}" +LABEL org.apache.airflow.airflow_stasd_exporter.version="${AIRFLOW_STATSD_EXPORTER_VERSION}" +LABEL org.apache.airflow.commit_sha="${COMMIT_SHA}" + +MAINTAINER "Apache Airflow Community " + +COPY mappings.yml /etc/statsd-exporter/mappings.yml diff --git a/chart/dockerfiles/statsd-exporter/build_and_push.sh b/chart/dockerfiles/statsd-exporter/build_and_push.sh new file mode 100755 index 0000000000000..e936fbd76b410 --- /dev/null +++ b/chart/dockerfiles/statsd-exporter/build_and_push.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +set -euo pipefail +DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} +DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"} +STATSD_VERSION="v0.17.0" +AIRFLOW_STATSD_EXPORTER_VERSION="2020.07.10" +COMMIT_SHA=$(git rev-parse HEAD) + +cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1 + +TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:airflow-statsd-exporter-${AIRFLOW_STATSD_EXPORTER_VERSION}-${STATSD_VERSION}" + +docker build . \ + --pull \ + --build-arg "STATSD_VERSION=${STATSD_VERSION}" \ + --build-arg "AIRFLOW_STATSD_EXPORTER_VERSION=${AIRFLOW_STATSD_EXPORTER_VERSION}" \ + --build-arg "COMMIT_SHA=${COMMIT_SHA}" \ + --tag "${TAG}" + +docker push "${TAG}" diff --git a/chart/dockerfiles/statsd-exporter/mappings.yml b/chart/dockerfiles/statsd-exporter/mappings.yml new file mode 100644 index 0000000000000..f86854e2188fe --- /dev/null +++ b/chart/dockerfiles/statsd-exporter/mappings.yml @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +mappings: + # Map dot separated stats to labels + - match: airflow.dagrun.dependency-check.*.* + name: "airflow_dagrun_dependency_check" + labels: + dag_id: "$1" + + - match: airflow.operator_successes_(.*) + match_type: regex + name: "airflow_operator_successes" + labels: + operator: "$1" + + - match: airflow.operator_failures_(.*) + match_type: regex + name: "airflow_operator_failures" + labels: + operator: "$1" + + - match: airflow.scheduler_heartbeat + match_type: regex + name: "airflow_scheduler_heartbeat" + labels: + type: counter + + - match: airflow.dag.*.*.duration + name: "airflow_task_duration" + labels: + dag_id: "$1" + task_id: "$2" + + - match: airflow.dagrun.duration.success.* + name: "airflow_dagrun_duration" + labels: + dag_id: "$1" + + - match: airflow.dagrun.duration.failed.* + name: "airflow_dagrun_failed" + labels: + dag_id: "$1" + + - match: airflow.dagrun.schedule_delay.* + name: "airflow_dagrun_schedule_delay" + labels: + dag_id: "$1" + + - match: airflow.dag_processing.last_runtime.* + name: "airflow_dag_processing_last_runtime" + labels: + dag_file: "$1" + + - match: airflow.dag_processing.last_run.seconds_ago.* + name: "airflow_dag_processing_last_run_seconds_ago" + labels: + dag_file: "$1" + + - match: airflow.pool.open_slots.* + name: "airflow_pool_open_slots" + labels: + pool: "$1" + + - match: airflow.pool.used_slots.* + name: "airflow_pool_used_slots" + labels: + pool: "$1" + + - match: airflow.pool.starving_tasks.* + name: "airflow_pool_starving_tasks" + labels: + pool: "$1" diff --git a/chart/values.yaml b/chart/values.yaml index 92fa4b9ed412c..150688befa344 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -128,20 +128,20 @@ images: tag: ~ pullPolicy: IfNotPresent statsd: - repository: astronomerinc/ap-statsd-exporter - tag: 0.11.0 + repository: apache/airflow + tag: airflow-statsd-exporter-2020.07.10-v0.17.0 pullPolicy: IfNotPresent redis: repository: redis tag: 6-buster pullPolicy: IfNotPresent pgbouncer: - repository: astronomerinc/ap-pgbouncer - tag: 1.8.1 + repository: apache/airflow + tag: airflow-pgbouncer-2020.07.10-1.14.0 pullPolicy: IfNotPresent pgbouncerExporter: - repository: astronomerinc/ap-pgbouncer-exporter - tag: 0.5.0-1 + repository: apache/airflow + tag: airflow-pgbouncer-exporter-2020.07.10-0.5.0 pullPolicy: IfNotPresent # Environment variables for all airflow containers diff --git a/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh b/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh index 5349b0f9d3cbb..d6d653dbe43a4 100755 --- a/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh +++ b/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh @@ -17,7 +17,6 @@ # under the License. # shellcheck source=scripts/ci/libraries/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" - set -euo pipefail export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:="3.6"}