From 23525c4cbda1bd12d1aac44bd2d784d11873c7e2 Mon Sep 17 00:00:00 2001
From: Dan Williams <dan@ithium.net>
Date: Wed, 10 Jul 2019 16:22:49 +0100
Subject: [PATCH 1/3] Added /mounts/logs/ location to save airflow logs for
 debugging (#1019)

---
 flowetl/mounts/logs/README.md         | 1 +
 flowetl/tests/integration/conftest.py | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)
 create mode 100644 flowetl/mounts/logs/README.md

diff --git a/flowetl/mounts/logs/README.md b/flowetl/mounts/logs/README.md
new file mode 100644
index 0000000000..5ee9760c06
--- /dev/null
+++ b/flowetl/mounts/logs/README.md
@@ -0,0 +1 @@
+# Logs location
diff --git a/flowetl/tests/integration/conftest.py b/flowetl/tests/integration/conftest.py
index 1154e4ad10..d56291c354 100644
--- a/flowetl/tests/integration/conftest.py
+++ b/flowetl/tests/integration/conftest.py
@@ -159,7 +159,8 @@ def mounts(postgres_data_dir_for_tests, flowetl_mounts_dir):
     """
     config_mount = Mount("/mounts/config", f"{flowetl_mounts_dir}/config", type="bind")
     files_mount = Mount("/mounts/files", f"{flowetl_mounts_dir}/files", type="bind")
-    flowetl_mounts = [config_mount, files_mount]
+    logs_mount = Mount("/mounts/logs", f"{flowetl_mounts_dir}/logs", type="bind")
+    flowetl_mounts = [config_mount, files_mount, logs_mount]
 
     data_mount = Mount(
         "/var/lib/postgresql/data", postgres_data_dir_for_tests, type="bind"

From 374180cfefbdc3a926f650bd3a5077d20cd11615 Mon Sep 17 00:00:00 2001
From: Dan Williams <dan@ithium.net>
Date: Wed, 10 Jul 2019 16:41:09 +0100
Subject: [PATCH 2/3] Added logic to save the airflow logs when env var present
 (#1019)

---
 flowetl/tests/integration/conftest.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/flowetl/tests/integration/conftest.py b/flowetl/tests/integration/conftest.py
index d56291c354..073dfab32c 100644
--- a/flowetl/tests/integration/conftest.py
+++ b/flowetl/tests/integration/conftest.py
@@ -102,6 +102,7 @@ def container_env():
         "AIRFLOW__CORE__FERNET_KEY": "ssgBqImdmQamCrM9jNhxI_IXSzvyVIfqvyzES67qqVU=",
         "AIRFLOW__CORE__SQL_ALCHEMY_CONN": f"postgres://{flowetl_db['POSTGRES_USER']}:{flowetl_db['POSTGRES_PASSWORD']}@flowetl_db:5432/{flowetl_db['POSTGRES_DB']}",
         "AIRFLOW_CONN_FLOWDB": f"postgres://{flowdb['POSTGRES_USER']}:{flowdb['POSTGRES_PASSWORD']}@flowdb:5432/flowdb",
+        "AIRFLOW_HOME": os.environ["AIRFLOW_HOME"],
         "POSTGRES_USER": "flowetl",
         "POSTGRES_PASSWORD": "flowetl",
         "POSTGRES_DB": "flowetl",
@@ -329,6 +330,22 @@ def wait_for_container(
     )
     wait_for_container()
     yield container
+
+    save_airflow_logs = (
+        os.getenv("FLOWETL_INTEGRATION_TESTS_SAVE_AIRFLOW_LOGS", "FALSE").upper()
+        == "TRUE"
+    )
+    if save_airflow_logs:
+        logger.info(
+            "Saving airflow logs to /mounts/logs/ and outputting to stdout "
+            "(because FLOWETL_INTEGRATION_TESTS_SAVE_AIRFLOW_LOGS=TRUE)."
+        )
+        container.exec_run("bash -c 'cp -r $AIRFLOW_HOME/logs/* /mounts/logs/'")
+        airflow_logs = container.exec_run(
+            "bash -c 'find /mounts/logs -type f -exec cat {} \;'"
+        )
+        logger.info(airflow_logs)
+
     container.kill()
     container.remove()
 

From 963f9c00221bf1dc1a9bd064a03c6d30e81d112b Mon Sep 17 00:00:00 2001
From: Dan Williams <dan@ithium.net>
Date: Wed, 10 Jul 2019 16:55:56 +0100
Subject: [PATCH 3/3] Added changelog entry (#1019)

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6f204e8d11..9909d68b92 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - FlowETL now supports ingesting from a postgres table in addition to CSV files. [#1027](https://github.com/Flowminder/FlowKit/issues/1027)
 - `FLOWETL_RUNTIME_CONFIG` environment variable added to control which DAG definitions the FlowETL integration tests should use (valid values: "testing", "production").
 - `FLOWETL_INTEGRATION_TESTS_DISABLE_PULLING_DOCKER_IMAGES` environment variable added to allow running the FlowETL integration tests against locally built docker images during development.
+- `FLOWETL_INTEGRATION_TESTS_SAVE_AIRFLOW_LOGS` environment variable added to allow copying the Airflow logs in FlowETL integration tests into the /mounts/logs directory for debugging. [#1019](https://github.com/Flowminder/FlowKit/issues/1019)
 
 ### Changed