From 6930905b172ef7f96b70e44f596582a2634f2062 Mon Sep 17 00:00:00 2001 From: resdev Date: Tue, 28 Apr 2020 18:32:44 -0700 Subject: [PATCH 1/4] Docker Compose CeleryExecutor example --- .../docker-compose/CeleryExecutor/.gitignore | 5 + .../docker-compose/CeleryExecutor/README.md | 27 ++++++ .../airflow-data/dags/test-dag.py | 80 ++++++++++++++++ .../CeleryExecutor/airflow-data/logs/test.log | 0 .../airflow-data/plugins/__init__.py | 0 .../CeleryExecutor/docker-compose.yml | 96 +++++++++++++++++++ .../CeleryExecutor/init_airflow_setup.sh | 20 ++++ 7 files changed, 228 insertions(+) create mode 100644 templates/docker-compose/CeleryExecutor/.gitignore create mode 100644 templates/docker-compose/CeleryExecutor/README.md create mode 100644 templates/docker-compose/CeleryExecutor/airflow-data/dags/test-dag.py create mode 100644 templates/docker-compose/CeleryExecutor/airflow-data/logs/test.log create mode 100644 templates/docker-compose/CeleryExecutor/airflow-data/plugins/__init__.py create mode 100644 templates/docker-compose/CeleryExecutor/docker-compose.yml create mode 100644 templates/docker-compose/CeleryExecutor/init_airflow_setup.sh diff --git a/templates/docker-compose/CeleryExecutor/.gitignore b/templates/docker-compose/CeleryExecutor/.gitignore new file mode 100644 index 0000000000000..aa494903821c8 --- /dev/null +++ b/templates/docker-compose/CeleryExecutor/.gitignore @@ -0,0 +1,5 @@ +# Created by .ignore support plugin (hsz.mobi) +.idea +airflow-data/logs +airflow-data/dags/__pycache__/ + diff --git a/templates/docker-compose/CeleryExecutor/README.md b/templates/docker-compose/CeleryExecutor/README.md new file mode 100644 index 0000000000000..9f390b962f384 --- /dev/null +++ b/templates/docker-compose/CeleryExecutor/README.md @@ -0,0 +1,27 @@ +# docker-compose-airflow +Docker Compose Apache Airflow (Official Docker Images) with CeleryExecutor, InitDB and InitUser + +Ideal for local development or small scale personal deployments. + +### How to deploy: + +**Step 1:** Clone this Repo and open terminal + +**Step 2:** Go through .env file, init_airflow_setup.sh, docker-compose.yml file to change settings according to your preference. Or you can just keep them as it is for local development. + +**Step 3:** Run `docker-compose up -d` + +**Step 4:** Run `sh init_airflow_setup.sh` (Run this only for initial deployment, Airflow container will be in restart mode till this script successfully executed.) + +**Step 5:** Go to http://localhost:8080 and login with user: _airflow_test_user_ and password: _airflow_test_password_ as specified in init_airflow_setup.sh script + +**Step 6:** Enable and Run few dags and monitor Celery workers at http://localhost:5555 + + +Airflow Environment variables are maintained in .env file where you can modify Airflow Config as specified in the link https://airflow.apache.org/docs/stable/howto/set-config.html + +You can modify Executor modes by setting value of .env variable - AIRFLOW__CORE__EXECUTOR=CeleryExecutor + +If you have existing airflow db, you can connect to it by setting .env variable - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:postgres@postgres:5432/airflow +(If you are using an existing Airflow DB. Do NOT run init_airflow_setup.sh) + diff --git a/templates/docker-compose/CeleryExecutor/airflow-data/dags/test-dag.py b/templates/docker-compose/CeleryExecutor/airflow-data/dags/test-dag.py new file mode 100644 index 0000000000000..31b36edb065f1 --- /dev/null +++ b/templates/docker-compose/CeleryExecutor/airflow-data/dags/test-dag.py @@ -0,0 +1,80 @@ +from datetime import timedelta +# The DAG object; we'll need this to instantiate a DAG +from airflow import DAG +# Operators; we need this to operate! +from airflow.operators.bash_operator import BashOperator +from airflow.utils.dates import days_ago +# These args will get passed on to each operator +# You can override them on a per-task basis during operator initialization + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': days_ago(2), + 'email': ['airflow@example.com'], + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), + # 'queue': 'bash_queue', + # 'pool': 'backfill', + # 'priority_weight': 10, + # 'end_date': datetime(2016, 1, 1), + # 'wait_for_downstream': False, + # 'dag': dag, + # 'sla': timedelta(hours=2), + # 'execution_timeout': timedelta(seconds=300), + # 'on_failure_callback': some_function, + # 'on_success_callback': some_other_function, + # 'on_retry_callback': another_function, + # 'sla_miss_callback': yet_another_function, + # 'trigger_rule': 'all_success' +} +dag = DAG( + 'tutorial', + default_args=default_args, + description='A simple tutorial DAG', + schedule_interval=timedelta(days=1), +) + +# t1, t2 and t3 are examples of tasks created by instantiating operators +t1 = BashOperator( + task_id='print_date', + bash_command='date', + dag=dag, +) + +t2 = BashOperator( + task_id='sleep', + depends_on_past=False, + bash_command='sleep 5', + retries=3, + dag=dag, +) +dag.doc_md = __doc__ + +t1.doc_md = """\ +#### Task Documentation +You can document your task using the attributes `doc_md` (markdown), +`doc` (plain text), `doc_rst`, `doc_json`, `doc_yaml` which gets +rendered in the UI's Task Instance Details page. +![img](http://montcs.bloomu.edu/~bobmon/Semesters/2012-01/491/import%20soul.png) +""" +templated_command = """ +{% for i in range(5) %} + echo "{{ ds }}" + echo "{{ macros.ds_add(ds, 7)}}" + echo "{{ params.my_param }}" +{% endfor %} +""" + +t3 = BashOperator( + task_id='templated', + depends_on_past=False, + bash_command=templated_command, + params={'my_param': 'Parameter I passed in'}, + dag=dag, +) + +t1 >> [t2, t3] + diff --git a/templates/docker-compose/CeleryExecutor/airflow-data/logs/test.log b/templates/docker-compose/CeleryExecutor/airflow-data/logs/test.log new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/templates/docker-compose/CeleryExecutor/airflow-data/plugins/__init__.py b/templates/docker-compose/CeleryExecutor/airflow-data/plugins/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/templates/docker-compose/CeleryExecutor/docker-compose.yml b/templates/docker-compose/CeleryExecutor/docker-compose.yml new file mode 100644 index 0000000000000..37fe2fea94d66 --- /dev/null +++ b/templates/docker-compose/CeleryExecutor/docker-compose.yml @@ -0,0 +1,96 @@ +version: '3' +services: + postgres: + image: postgres:latest + container_name: postgres_cont + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=airflow + - POSTGRES_PORT=5432 + ports: + - 5432:5432 + + redis: + image: redis:latest + container_name: redis_cont + restart: always + ports: + - 6379:6379 + + airflow: + image: apache/airflow:1.10.10 + container_name: airflow_cont + env_file: + - .env + restart: always + command: webserver + ports: + - 8080:8080 + volumes: + - ./airflow-data/dags:/opt/airflow/dags + - ./airflow-data/logs:/opt/airflow/logs + - ./airflow-data/plugins:/opt/airflow/plugins + + airflow-scheduler: + image: apache/airflow:1.10.10 + container_name: airflow_scheduler_cont + env_file: + - .env + restart: always + command: scheduler + volumes: + - ./airflow-data/dags:/opt/airflow/dags + - ./airflow-data/logs:/opt/airflow/logs + - ./airflow-data/plugins:/opt/airflow/plugins + + airflow-worker1: + image: apache/airflow:1.10.10 + container_name: airflow_worker1_cont + env_file: + - .env + restart: always + command: worker + volumes: + - ./airflow-data/dags:/opt/airflow/dags + - ./airflow-data/logs:/opt/airflow/logs + - ./airflow-data/plugins:/opt/airflow/plugins + + airflow-worker2: + image: apache/airflow:1.10.10 + container_name: airflow_worker2_cont + env_file: + - .env + restart: always + command: worker + volumes: + - ./airflow-data/dags:/opt/airflow/dags + - ./airflow-data/logs:/opt/airflow/logs + - ./airflow-data/plugins:/opt/airflow/plugins + + airflow-worker3: + image: apache/airflow:1.10.10 + container_name: airflow_worker3_cont + env_file: + - .env + command: worker + restart: always + volumes: + - ./airflow-data/dags:/opt/airflow/dags + - ./airflow-data/logs:/opt/airflow/logs + - ./airflow-data/plugins:/opt/airflow/plugins + + airflow-flower: + image: apache/airflow:1.10.10 + container_name: airflow_flower_cont + restart: always + volumes: + - ./airflow-data/dags:/opt/airflow/dags + - ./airflow-data/logs:/opt/airflow/logs + - ./airflow-data/plugins:/opt/airflow/plugins + env_file: + - .env + command: flower + ports: + - 5555:5555 + diff --git a/templates/docker-compose/CeleryExecutor/init_airflow_setup.sh b/templates/docker-compose/CeleryExecutor/init_airflow_setup.sh new file mode 100644 index 0000000000000..cc2f38cc26cbe --- /dev/null +++ b/templates/docker-compose/CeleryExecutor/init_airflow_setup.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +# WARNING: Run this script only during initial db setup. DO NOT run this script on an existing Airflow DB. + +IS_INITDB=True +AIRFLOW_USER=airflow_test_user +AIRFLOW_PASSWORD=airflow_test_password +AIRFLOW_USER_EMAIL=airflow@airflow.com + +if [ $IS_INITDB ]; then + + echo "Initializing Airflow DB setup and Admin user setup because value of IS_INITDB is $IS_INITDB" + echo " Airflow admin username will be $AIRFLOW_USER" + + docker exec -ti airflow_cont airflow initdb && echo "Initialized airflow DB" + docker exec -ti airflow_cont airflow create_user --role Admin --username $AIRFLOW_USER --password $AIRFLOW_PASSWORD -e $AIRFLOW_USER_EMAIL -f airflow -l airflow && echo "Created airflow Initial admin user with username $AIRFLOW_USER" + +else + echo "Skipping InitDB and InitUser setup because value of IS_INITDB is $IS_INITDB" +fi From f05b39595d3d0ce1432773f2f4d3c22e58d7c11d Mon Sep 17 00:00:00 2001 From: resdev Date: Tue, 28 Apr 2020 18:43:54 -0700 Subject: [PATCH 2/4] Docker Compose CeleryExecutor example --- templates/docker-compose/CeleryExecutor/.gitignore | 1 - templates/docker-compose/CeleryExecutor/docker-compose.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/docker-compose/CeleryExecutor/.gitignore b/templates/docker-compose/CeleryExecutor/.gitignore index aa494903821c8..0dfa47517107c 100644 --- a/templates/docker-compose/CeleryExecutor/.gitignore +++ b/templates/docker-compose/CeleryExecutor/.gitignore @@ -2,4 +2,3 @@ .idea airflow-data/logs airflow-data/dags/__pycache__/ - diff --git a/templates/docker-compose/CeleryExecutor/docker-compose.yml b/templates/docker-compose/CeleryExecutor/docker-compose.yml index 37fe2fea94d66..0b5d2f2aae41d 100644 --- a/templates/docker-compose/CeleryExecutor/docker-compose.yml +++ b/templates/docker-compose/CeleryExecutor/docker-compose.yml @@ -68,6 +68,7 @@ services: - ./airflow-data/logs:/opt/airflow/logs - ./airflow-data/plugins:/opt/airflow/plugins + airflow-worker3: image: apache/airflow:1.10.10 container_name: airflow_worker3_cont From 01faeffac9c4cca1464d3c916dd2ad83fede46f4 Mon Sep 17 00:00:00 2001 From: resdev Date: Tue, 28 Apr 2020 18:56:01 -0700 Subject: [PATCH 3/4] Docker Compose CeleryExecutor example --- .../CeleryExecutor/airflow-env-variables.env | 18 ++++++++++++++++++ .../CeleryExecutor/docker-compose.yml | 12 ++++++------ 2 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 templates/docker-compose/CeleryExecutor/airflow-env-variables.env diff --git a/templates/docker-compose/CeleryExecutor/airflow-env-variables.env b/templates/docker-compose/CeleryExecutor/airflow-env-variables.env new file mode 100644 index 0000000000000..f4dd8764fbc85 --- /dev/null +++ b/templates/docker-compose/CeleryExecutor/airflow-env-variables.env @@ -0,0 +1,18 @@ + + +# AIRFLOW CORE + +AIRFLOW__CORE__EXECUTOR=CeleryExecutor +AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:postgres@postgres:5432/airflow +AIRFLOW__CORE__FERNET_KEY=FB0o_zt4e3Ziq3LdUUO7F2Z95cvFFx16hU8jTeR1ASM= +AIRFLOW__CORE__LOAD_EXAMPLES=True + +#AIRFLOW WEBSERVER +AIRFLOW__WEBSERVER__RBAC=True +AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True + + +# AIRFLOW CELERY +AIRFLOW__CELERY__BROKER_URL=redis://:@redis:6379/0 +AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://postgres:postgres@postgres:5432/airflow + diff --git a/templates/docker-compose/CeleryExecutor/docker-compose.yml b/templates/docker-compose/CeleryExecutor/docker-compose.yml index 0b5d2f2aae41d..7654bbce15b9d 100644 --- a/templates/docker-compose/CeleryExecutor/docker-compose.yml +++ b/templates/docker-compose/CeleryExecutor/docker-compose.yml @@ -22,7 +22,7 @@ services: image: apache/airflow:1.10.10 container_name: airflow_cont env_file: - - .env + - airflow-env-variables.env restart: always command: webserver ports: @@ -36,7 +36,7 @@ services: image: apache/airflow:1.10.10 container_name: airflow_scheduler_cont env_file: - - .env + - airflow-env-variables.env restart: always command: scheduler volumes: @@ -48,7 +48,7 @@ services: image: apache/airflow:1.10.10 container_name: airflow_worker1_cont env_file: - - .env + - airflow-env-variables.env restart: always command: worker volumes: @@ -60,7 +60,7 @@ services: image: apache/airflow:1.10.10 container_name: airflow_worker2_cont env_file: - - .env + - airflow-env-variables.env restart: always command: worker volumes: @@ -73,7 +73,7 @@ services: image: apache/airflow:1.10.10 container_name: airflow_worker3_cont env_file: - - .env + - airflow-env-variables.env command: worker restart: always volumes: @@ -90,7 +90,7 @@ services: - ./airflow-data/logs:/opt/airflow/logs - ./airflow-data/plugins:/opt/airflow/plugins env_file: - - .env + - airflow-env-variables.env command: flower ports: - 5555:5555 From 0282e7ac13daa2463be919644e4e94e029fed9eb Mon Sep 17 00:00:00 2001 From: resdev Date: Tue, 28 Apr 2020 19:10:28 -0700 Subject: [PATCH 4/4] readme update --- templates/docker-compose/CeleryExecutor/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/docker-compose/CeleryExecutor/README.md b/templates/docker-compose/CeleryExecutor/README.md index 9f390b962f384..a1d6cfa57753c 100644 --- a/templates/docker-compose/CeleryExecutor/README.md +++ b/templates/docker-compose/CeleryExecutor/README.md @@ -3,6 +3,8 @@ Docker Compose Apache Airflow (Official Docker Images) with CeleryExecutor, Init Ideal for local development or small scale personal deployments. +Prerequisites: Docker and docker-compose + ### How to deploy: **Step 1:** Clone this Repo and open terminal