From 0ca11ada7b93e5d7bcf44eeb838eee9a84bbb45f Mon Sep 17 00:00:00 2001 From: Thingus Date: Thu, 29 Feb 2024 09:59:53 +0000 Subject: [PATCH 01/31] Initial automatic running of all ingestion tests --- flowdb/testdata/test_data/py/run_etl_tests.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 flowdb/testdata/test_data/py/run_etl_tests.py diff --git a/flowdb/testdata/test_data/py/run_etl_tests.py b/flowdb/testdata/test_data/py/run_etl_tests.py new file mode 100644 index 0000000000..dc4f4f19c4 --- /dev/null +++ b/flowdb/testdata/test_data/py/run_etl_tests.py @@ -0,0 +1,105 @@ +from dataclasses import asdict, dataclass +from datetime import date, datetime +from itertools import product +from pathlib import Path +from typing import Generator +import flowetl +from jinja2 import Environment, PackageLoader, Template +from sqlalchemy import create_engine +import os + +env = Environment(loader=PackageLoader("flowetl", "qa_checks/qa_checks")) + +count_duplicate_template = env.get_template("count_duplicates.sql") + +count_duplicates = count_duplicate_template.render( + final_table="events.calls", cdr_type="events" +) + +db_user = os.getenv("POSTGRES_USER", "flowdb") +db_name = os.getenv("POSTGRES_DB", "flowdb") +db_port = os.getenv("POSTGRES_PORT", "9000") +db_password = os.getenv("POSTGRES_PASSWORD", "flowflow") # In here for dev, don't think +conn_str = f"postgresql://{db_user}:{db_password}@localhost:{db_port}/{db_name}" +print(conn_str) +print(count_duplicates) +engine = create_engine(conn_str) +out = engine.execute(count_duplicates) +print(out.fetchall()) + + +# @James; if something like this already exists, I'll grab +# that instead +update_template_string = """ +INSERT INTO etl.post_etl_queries + (cdr_date, cdr_type, type_of_query_or_check, outcome, optional_comment_or_description, timestamp) +VALUES( + '{{cdr_date}}', + '{{cdr_type}}', + '{{type_of_query_or_check}}', + ({{outcome_query}}), + '{{optional_comment_or_description}}', + '{{timestamp}}' +) + +""" + + +@dataclass +class QaTemplate: + display_name: str + template: Template + + +update_template = env.from_string(update_template_string) + + +@dataclass +class QaRow: + cdr_date: date + cdr_type: str + type_of_query_or_check: str + outcome_query: str + optional_comment_or_description: str + timestamp: datetime + + +@dataclass +class MockQaScenario: + dates: list[date] + tables: list[str] + + +qa_scn = MockQaScenario([date(2021, 1, 1), date(2021, 1, 2)], ["calls"]) + + +def render_qa_check(template: Template, date: date, cdr_type: str) -> str: + return template.render( + final_table=f"events.{cdr_type}", + cdr_type=cdr_type, + ds=date.strftime("%Y-%m-%d"), + ) + + +templates = ( + QaTemplate(Path(t).name, env.get_template(t)) for t in env.list_templates(".sql") +) + +qa_rows = ( + QaRow( + date, + type, + template.display_name, + render_qa_check(template.template, date, type), + "Made from mock data", + datetime.now(), + ) + for date, type, template in product(qa_scn.dates, qa_scn.tables, templates) +) + +with engine.connect() as conn: + for row in qa_rows: + conn.execute(update_template.render(**asdict(row))) + + out = conn.execute("SELECT * FROM etl.post_etl_queries") + print(out.fetchall()) From 537296c87eeab40fd149e034a19050b12455cdd3 Mon Sep 17 00:00:00 2001 From: Thingus Date: Thu, 29 Feb 2024 10:47:12 +0000 Subject: [PATCH 02/31] Tidying up qa_check script --- flowdb/testdata/test_data/py/run_etl_tests.py | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/flowdb/testdata/test_data/py/run_etl_tests.py b/flowdb/testdata/test_data/py/run_etl_tests.py index dc4f4f19c4..ceddd70fc5 100644 --- a/flowdb/testdata/test_data/py/run_etl_tests.py +++ b/flowdb/testdata/test_data/py/run_etl_tests.py @@ -3,29 +3,18 @@ from itertools import product from pathlib import Path from typing import Generator -import flowetl from jinja2 import Environment, PackageLoader, Template from sqlalchemy import create_engine import os env = Environment(loader=PackageLoader("flowetl", "qa_checks/qa_checks")) -count_duplicate_template = env.get_template("count_duplicates.sql") - -count_duplicates = count_duplicate_template.render( - final_table="events.calls", cdr_type="events" -) - db_user = os.getenv("POSTGRES_USER", "flowdb") db_name = os.getenv("POSTGRES_DB", "flowdb") db_port = os.getenv("POSTGRES_PORT", "9000") db_password = os.getenv("POSTGRES_PASSWORD", "flowflow") # In here for dev, don't think conn_str = f"postgresql://{db_user}:{db_password}@localhost:{db_port}/{db_name}" -print(conn_str) -print(count_duplicates) engine = create_engine(conn_str) -out = engine.execute(count_duplicates) -print(out.fetchall()) # @James; if something like this already exists, I'll grab @@ -44,6 +33,8 @@ """ +update_template = env.from_string(update_template_string) + @dataclass class QaTemplate: @@ -51,9 +42,6 @@ class QaTemplate: template: Template -update_template = env.from_string(update_template_string) - - @dataclass class QaRow: cdr_date: date @@ -70,9 +58,8 @@ class MockQaScenario: tables: list[str] -qa_scn = MockQaScenario([date(2021, 1, 1), date(2021, 1, 2)], ["calls"]) - - +# NOTE: given this gets run _after_ all the ingestion, 'dates' may be an irrelevence +# unless final_table is normally one of the dated partitions def render_qa_check(template: Template, date: date, cdr_type: str) -> str: return template.render( final_table=f"events.{cdr_type}", @@ -81,10 +68,13 @@ def render_qa_check(template: Template, date: date, cdr_type: str) -> str: ) +qa_scn = MockQaScenario(dates=[date(2016, 1, 1), date(2016, 1, 2)], tables=["calls"]) + templates = ( QaTemplate(Path(t).name, env.get_template(t)) for t in env.list_templates(".sql") ) + qa_rows = ( QaRow( date, From fd2fef93bd6071e5239d647f1d9d51a538391e24 Mon Sep 17 00:00:00 2001 From: Thingus Date: Thu, 29 Feb 2024 10:59:30 +0000 Subject: [PATCH 03/31] Moving qa check script to testdata/bin --- .../{test_data/py/run_etl_tests.py => bin/run_qa_checks.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename flowdb/testdata/{test_data/py/run_etl_tests.py => bin/run_qa_checks.py} (100%) diff --git a/flowdb/testdata/test_data/py/run_etl_tests.py b/flowdb/testdata/bin/run_qa_checks.py similarity index 100% rename from flowdb/testdata/test_data/py/run_etl_tests.py rename to flowdb/testdata/bin/run_qa_checks.py From 2e248a63242ac35761de8d1f33cd46ffacac0d54 Mon Sep 17 00:00:00 2001 From: Thingus Date: Thu, 29 Feb 2024 13:38:15 +0000 Subject: [PATCH 04/31] Argparse in run_qa_checks --- flowdb/testdata/bin/run_qa_checks.py | 77 +++++++++++++++++----------- 1 file changed, 47 insertions(+), 30 deletions(-) diff --git a/flowdb/testdata/bin/run_qa_checks.py b/flowdb/testdata/bin/run_qa_checks.py index ceddd70fc5..d8825c9a78 100644 --- a/flowdb/testdata/bin/run_qa_checks.py +++ b/flowdb/testdata/bin/run_qa_checks.py @@ -2,20 +2,13 @@ from datetime import date, datetime from itertools import product from pathlib import Path -from typing import Generator from jinja2 import Environment, PackageLoader, Template from sqlalchemy import create_engine import os +import argparse -env = Environment(loader=PackageLoader("flowetl", "qa_checks/qa_checks")) - -db_user = os.getenv("POSTGRES_USER", "flowdb") -db_name = os.getenv("POSTGRES_DB", "flowdb") -db_port = os.getenv("POSTGRES_PORT", "9000") -db_password = os.getenv("POSTGRES_PASSWORD", "flowflow") # In here for dev, don't think -conn_str = f"postgresql://{db_user}:{db_password}@localhost:{db_port}/{db_name}" -engine = create_engine(conn_str) +env = Environment(loader=PackageLoader("flowetl", "qa_checks/qa_checks")) # @James; if something like this already exists, I'll grab # that instead @@ -68,28 +61,52 @@ def render_qa_check(template: Template, date: date, cdr_type: str) -> str: ) -qa_scn = MockQaScenario(dates=[date(2016, 1, 1), date(2016, 1, 2)], tables=["calls"]) - -templates = ( - QaTemplate(Path(t).name, env.get_template(t)) for t in env.list_templates(".sql") -) - +if __name__ == "__main__()": + parser = argparse.ArgumentParser( + description="Runs all flowetl checks for ingested data" + ) + parser.add_argument( + "--dates", + type=lambda s: datetime.datetime.strptime(s, "%Y-%m-%d"), + help="Date to run ingestion check on. Can be specified multiple times.", + nargs="+", + ) + parser.add_argument( + "--event_types", help="Event tables to run qa checks on.", nargs="+" + ) + args = parser.parse_args() + + db_user = os.getenv("POSTGRES_USER", "flowdb") + db_name = os.getenv("POSTGRES_DB", "flowdb") + db_port = os.getenv("POSTGRES_PORT", "9000") + db_password = os.getenv( + "POSTGRES_PASSWORD", "flowflow" + ) # In here for dev, don't think we'll need it in prod + conn_str = f"postgresql://{db_user}:{db_password}@localhost:{db_port}/{db_name}" + engine = create_engine(conn_str) + + qa_scn = MockQaScenario(dates=args.dates, tables=args.tables) + + templates = ( + QaTemplate(Path(t).name, env.get_template(t)) + for t in env.list_templates(".sql") + ) -qa_rows = ( - QaRow( - date, - type, - template.display_name, - render_qa_check(template.template, date, type), - "Made from mock data", - datetime.now(), + qa_rows = ( + QaRow( + date, + type, + template.display_name, + render_qa_check(template.template, date, type), + "Made from mock data", + datetime.now(), + ) + for date, type, template in product(qa_scn.dates, qa_scn.tables, templates) ) - for date, type, template in product(qa_scn.dates, qa_scn.tables, templates) -) -with engine.connect() as conn: - for row in qa_rows: - conn.execute(update_template.render(**asdict(row))) + with engine.connect() as conn: + for row in qa_rows: + conn.execute(update_template.render(**asdict(row))) - out = conn.execute("SELECT * FROM etl.post_etl_queries") - print(out.fetchall()) + out = conn.execute("SELECT * FROM etl.post_etl_queries") + print(out.fetchall()) From 210fc5f1dc208b58638773285faa7563f5724f6d Mon Sep 17 00:00:00 2001 From: Thingus Date: Thu, 29 Feb 2024 16:16:00 +0000 Subject: [PATCH 05/31] Adding to ingestion scripts --- flowdb/testdata/bin/9900_ingest_synthetic_data.sh | 3 +++ flowdb/testdata/bin/9900_ingest_test_data.sh | 4 ++++ flowdb/testdata/bin/run_qa_checks.py | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh index d984539753..7bedb52cac 100644 --- a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh +++ b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh @@ -76,3 +76,6 @@ else echo "Must set SYNTHETIC_DATA_GENERATOR environment variable to 'sql' or 'python'." exit 1 fi +if [ "$TEST_QA_CHECK" ]; then + pipenv run python ./run_qa_checks.py --dates ${DISASTER_END:-"2015-01-01"} --event-types calls mds sms +fi diff --git a/flowdb/testdata/bin/9900_ingest_test_data.sh b/flowdb/testdata/bin/9900_ingest_test_data.sh index d387f18c3f..17f10e714a 100644 --- a/flowdb/testdata/bin/9900_ingest_test_data.sh +++ b/flowdb/testdata/bin/9900_ingest_test_data.sh @@ -31,3 +31,7 @@ if [ $count != 0 ]; then echo "$DIR is empty." fi fi + +if [ "$TEST_QA_CHECK" ]; then + pipenv run python ./run_qa_checks.py --dates ${DISASTER_END:-"2015-01-01"} --event-types calls mds sms +fi \ No newline at end of file diff --git a/flowdb/testdata/bin/run_qa_checks.py b/flowdb/testdata/bin/run_qa_checks.py index d8825c9a78..1cd0cb8ac9 100644 --- a/flowdb/testdata/bin/run_qa_checks.py +++ b/flowdb/testdata/bin/run_qa_checks.py @@ -72,7 +72,7 @@ def render_qa_check(template: Template, date: date, cdr_type: str) -> str: nargs="+", ) parser.add_argument( - "--event_types", help="Event tables to run qa checks on.", nargs="+" + "--event-types", help="Event tables to run qa checks on.", nargs="+" ) args = parser.parse_args() From b1b3c46e3431b61eb29e70f77d0895a2cc0aab2a Mon Sep 17 00:00:00 2001 From: Thingus Date: Fri, 1 Mar 2024 10:22:40 +0000 Subject: [PATCH 06/31] Adding QA check to circleci to check build --- .circleci/config.yml | 1 + flowdb/testdata/bin/run_qa_checks.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 60a0616371..d7fb66f087 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,6 +35,7 @@ defaults: FLOWAPI_FLOWDB_USER: flowapi FLOWMACHINE_FLOWDB_PASSWORD: foo FLOWAPI_FLOWDB_PASSWORD: foo + TEST_QA_CHECK: 1 - &wait_for_flowdb name: Wait for flowdb to start command: | diff --git a/flowdb/testdata/bin/run_qa_checks.py b/flowdb/testdata/bin/run_qa_checks.py index 1cd0cb8ac9..14d5f13831 100644 --- a/flowdb/testdata/bin/run_qa_checks.py +++ b/flowdb/testdata/bin/run_qa_checks.py @@ -2,6 +2,7 @@ from datetime import date, datetime from itertools import product from pathlib import Path +from typing import List from jinja2 import Environment, PackageLoader, Template from sqlalchemy import create_engine import os @@ -47,8 +48,8 @@ class QaRow: @dataclass class MockQaScenario: - dates: list[date] - tables: list[str] + dates: List[date] + tables: List[str] # NOTE: given this gets run _after_ all the ingestion, 'dates' may be an irrelevence From ffbccc5b4118cd0bcf9d440fdac153474ee58997 Mon Sep 17 00:00:00 2001 From: Thingus Date: Mon, 11 Mar 2024 09:52:40 +0000 Subject: [PATCH 07/31] Update flowdb/testdata/bin/run_qa_checks.py Co-authored-by: Jonathan Gray --- flowdb/testdata/bin/run_qa_checks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flowdb/testdata/bin/run_qa_checks.py b/flowdb/testdata/bin/run_qa_checks.py index 14d5f13831..b13b806d91 100644 --- a/flowdb/testdata/bin/run_qa_checks.py +++ b/flowdb/testdata/bin/run_qa_checks.py @@ -1,3 +1,7 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + from dataclasses import asdict, dataclass from datetime import date, datetime from itertools import product From 774b4144648e111bbdb1d1f7daf906fdae2b5c2e Mon Sep 17 00:00:00 2001 From: Thingus Date: Mon, 11 Mar 2024 11:51:37 +0000 Subject: [PATCH 08/31] Comments from review --- development_environment | 1 + flowdb/Pipfile | 1 + flowdb/testdata/bin/9900_ingest_synthetic_data.sh | 3 ++- flowdb/testdata/bin/9900_ingest_test_data.sh | 5 +++-- flowdb/testdata/bin/run_qa_checks.py | 12 ++++-------- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/development_environment b/development_environment index d1da73e11c..6f20ece4c9 100644 --- a/development_environment +++ b/development_environment @@ -122,6 +122,7 @@ SUBSCRIBERS_SEED=12345 CALLS_SEED=22222 CELLS_SEED=99999 OUTPUT_ROOT_DIR=/docker-entrypoint-initdb.d +SKIP_TEST_QA_CHECK=False # Integration tests diff --git a/flowdb/Pipfile b/flowdb/Pipfile index 0df894d0e1..be0b08adf8 100644 --- a/flowdb/Pipfile +++ b/flowdb/Pipfile @@ -8,6 +8,7 @@ psutil = "*" [dev-packages] black = {extras = ["jupyter"],version = "==24.2.0"} +jinja2 = "*" [requires] python_version = "3.9" diff --git a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh index 7bedb52cac..fd10979853 100644 --- a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh +++ b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh @@ -76,6 +76,7 @@ else echo "Must set SYNTHETIC_DATA_GENERATOR environment variable to 'sql' or 'python'." exit 1 fi -if [ "$TEST_QA_CHECK" ]; then +if [ "${SKIP_TEST_QA_CHECK,,}" != "true" ]; then + echo "Running QA checks on test data" pipenv run python ./run_qa_checks.py --dates ${DISASTER_END:-"2015-01-01"} --event-types calls mds sms fi diff --git a/flowdb/testdata/bin/9900_ingest_test_data.sh b/flowdb/testdata/bin/9900_ingest_test_data.sh index 17f10e714a..5f67b8953d 100644 --- a/flowdb/testdata/bin/9900_ingest_test_data.sh +++ b/flowdb/testdata/bin/9900_ingest_test_data.sh @@ -32,6 +32,7 @@ if [ $count != 0 ]; then fi fi -if [ "$TEST_QA_CHECK" ]; then +# &{VAR,,} should lowercase the variable on interpolation +if [ "${SKIP_TEST_QA_CHECK,,}" != "true" ]; then pipenv run python ./run_qa_checks.py --dates ${DISASTER_END:-"2015-01-01"} --event-types calls mds sms -fi \ No newline at end of file +fi diff --git a/flowdb/testdata/bin/run_qa_checks.py b/flowdb/testdata/bin/run_qa_checks.py index b13b806d91..cc9593b8b7 100644 --- a/flowdb/testdata/bin/run_qa_checks.py +++ b/flowdb/testdata/bin/run_qa_checks.py @@ -15,8 +15,6 @@ env = Environment(loader=PackageLoader("flowetl", "qa_checks/qa_checks")) -# @James; if something like this already exists, I'll grab -# that instead update_template_string = """ INSERT INTO etl.post_etl_queries (cdr_date, cdr_type, type_of_query_or_check, outcome, optional_comment_or_description, timestamp) @@ -56,8 +54,6 @@ class MockQaScenario: tables: List[str] -# NOTE: given this gets run _after_ all the ingestion, 'dates' may be an irrelevence -# unless final_table is normally one of the dated partitions def render_qa_check(template: Template, date: date, cdr_type: str) -> str: return template.render( final_table=f"events.{cdr_type}", @@ -100,16 +96,16 @@ def render_qa_check(template: Template, date: date, cdr_type: str) -> str: qa_rows = ( QaRow( date, - type, + cdr_type, template.display_name, - render_qa_check(template.template, date, type), + render_qa_check(template.template, date, cdr_type), "Made from mock data", datetime.now(), ) - for date, type, template in product(qa_scn.dates, qa_scn.tables, templates) + for date, cdr_type, template in product(qa_scn.dates, qa_scn.tables, templates) ) - with engine.connect() as conn: + with engine.begin() as conn: for row in qa_rows: conn.execute(update_template.render(**asdict(row))) From 2b9e843f6431ecbd3b867c00235ceed85d5b6e2d Mon Sep 17 00:00:00 2001 From: Thingus Date: Mon, 11 Mar 2024 11:55:30 +0000 Subject: [PATCH 09/31] Relocking pipfile --- flowdb/Pipfile.lock | 93 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 79 insertions(+), 14 deletions(-) diff --git a/flowdb/Pipfile.lock b/flowdb/Pipfile.lock index b24bbe9aac..6c98f15334 100644 --- a/flowdb/Pipfile.lock +++ b/flowdb/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "53ce8e0a1b6ff7e660a3205ca5d0ae5e4fe2dbd5291dc5e6efe65bacf003c96c" + "sha256": "b6d55af1291d49a821d9079dc4aeed29b8b9f64d88ded1a77ab6ac793f40105a" }, "pipfile-spec": 6, "requires": { @@ -36,19 +36,10 @@ "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8" ], "index": "pypi", - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==5.9.8" } }, "develop": { - "appnope": { - "hashes": [ - "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee", - "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c" - ], - "markers": "sys_platform == 'darwin'", - "version": "==0.1.4" - }, "asttokens": { "hashes": [ "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24", @@ -91,7 +82,7 @@ "sha256:e8a6ae970537e67830776488bca52000eaa37fa63b9988e8c487458d9cd5ace6", "sha256:faf2ee02e6612577ba0181f4347bcbcf591eb122f7841ae5ba233d12c39dcb4d" ], - "markers": "python_version >= '3.8'", + "index": "pypi", "version": "==24.2.0" }, "click": { @@ -133,6 +124,80 @@ "markers": "python_version >= '3.6'", "version": "==0.19.1" }, + "jinja2": { + "hashes": [ + "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa", + "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90" + ], + "index": "pypi", + "version": "==3.1.3" + }, + "markupsafe": { + "hashes": [ + "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf", + "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff", + "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f", + "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3", + "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532", + "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f", + "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617", + "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df", + "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4", + "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906", + "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f", + "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4", + "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8", + "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371", + "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2", + "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465", + "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52", + "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6", + "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169", + "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad", + "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2", + "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0", + "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029", + "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f", + "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a", + "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced", + "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5", + "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c", + "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf", + "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9", + "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb", + "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad", + "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3", + "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1", + "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46", + "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc", + "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a", + "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee", + "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900", + "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5", + "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea", + "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f", + "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5", + "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e", + "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a", + "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f", + "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50", + "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a", + "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b", + "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4", + "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff", + "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2", + "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46", + "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b", + "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf", + "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5", + "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5", + "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab", + "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd", + "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68" + ], + "markers": "python_version >= '3.7'", + "version": "==2.1.5" + }, "matplotlib-inline": { "hashes": [ "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311", @@ -151,11 +216,11 @@ }, "packaging": { "hashes": [ - "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", - "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" + "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", + "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9" ], "markers": "python_version >= '3.7'", - "version": "==23.2" + "version": "==24.0" }, "parso": { "hashes": [ From 26a53e2ab3ce5404bd73324dbeb9822f9104bfcb Mon Sep 17 00:00:00 2001 From: Thingus Date: Mon, 11 Mar 2024 11:57:33 +0000 Subject: [PATCH 10/31] CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c99be29f8b..0c4f211e37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added +- Test and synthetic data generators now perform QA checks on the generated data. [#6467](https://github.com/Flowminder/FlowKit/issues/6467) ### Changed - `MostFrequentLocation` now breaks ties based on the last used location, instead of by arbitrary Postgres sort order. [#6268](https://github.com/Flowminder/FlowKit/issues/6268) From 8a811413668f9b511827a61d92f51510c34fd764 Mon Sep 17 00:00:00 2001 From: Thingus Date: Tue, 12 Mar 2024 09:07:17 +0000 Subject: [PATCH 11/31] Shifting jinja to test pipenv + relocking --- flowdb/Pipfile | 1 - flowdb/Pipfile.lock | 76 +-------- flowdb/testdata/test_data/Pipfile | 1 + flowdb/testdata/test_data/Pipfile.lock | 210 +++++++++++++++++-------- 4 files changed, 143 insertions(+), 145 deletions(-) diff --git a/flowdb/Pipfile b/flowdb/Pipfile index be0b08adf8..0df894d0e1 100644 --- a/flowdb/Pipfile +++ b/flowdb/Pipfile @@ -8,7 +8,6 @@ psutil = "*" [dev-packages] black = {extras = ["jupyter"],version = "==24.2.0"} -jinja2 = "*" [requires] python_version = "3.9" diff --git a/flowdb/Pipfile.lock b/flowdb/Pipfile.lock index 6c98f15334..c13328f06f 100644 --- a/flowdb/Pipfile.lock +++ b/flowdb/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "b6d55af1291d49a821d9079dc4aeed29b8b9f64d88ded1a77ab6ac793f40105a" + "sha256": "53ce8e0a1b6ff7e660a3205ca5d0ae5e4fe2dbd5291dc5e6efe65bacf003c96c" }, "pipfile-spec": 6, "requires": { @@ -124,80 +124,6 @@ "markers": "python_version >= '3.6'", "version": "==0.19.1" }, - "jinja2": { - "hashes": [ - "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa", - "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90" - ], - "index": "pypi", - "version": "==3.1.3" - }, - "markupsafe": { - "hashes": [ - "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf", - "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff", - "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f", - "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3", - "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532", - "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f", - "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617", - "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df", - "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4", - "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906", - "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f", - "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4", - "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8", - "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371", - "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2", - "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465", - "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52", - "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6", - "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169", - "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad", - "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2", - "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0", - "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029", - "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f", - "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a", - "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced", - "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5", - "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c", - "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf", - "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9", - "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb", - "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad", - "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3", - "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1", - "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46", - "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc", - "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a", - "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee", - "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900", - "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5", - "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea", - "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f", - "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5", - "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e", - "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a", - "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f", - "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50", - "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a", - "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b", - "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4", - "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff", - "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2", - "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46", - "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b", - "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf", - "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5", - "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5", - "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab", - "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd", - "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68" - ], - "markers": "python_version >= '3.7'", - "version": "==2.1.5" - }, "matplotlib-inline": { "hashes": [ "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311", diff --git a/flowdb/testdata/test_data/Pipfile b/flowdb/testdata/test_data/Pipfile index 1ce6197c77..43b0d197de 100644 --- a/flowdb/testdata/test_data/Pipfile +++ b/flowdb/testdata/test_data/Pipfile @@ -7,6 +7,7 @@ name = "pypi" tohu = "*" "geoalchemy2" = "*" sqlalchemy = "*" +jinja2 = "*" [dev-packages] black = {extras = ["jupyter"],version = "==24.2.0"} diff --git a/flowdb/testdata/test_data/Pipfile.lock b/flowdb/testdata/test_data/Pipfile.lock index 145092049f..542c21648c 100644 --- a/flowdb/testdata/test_data/Pipfile.lock +++ b/flowdb/testdata/test_data/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c515ca8f7a95e69361993fa60abfb5a0b1b4fce310d837d7ca74e99cddbb7478" + "sha256": "519103b63d51b0b041c5e5b8bee26aec6dfbc426f55f181e820c130bb074a47c" }, "pipfile-spec": 6, "requires": { @@ -34,20 +34,19 @@ }, "faker": { "hashes": [ - "sha256:0520a6b97e07c658b2798d7140971c1d5bc4bcd5013e7937fe075fd054aa320c", - "sha256:f07b64d27f67b62c7f0536a72f47813015b3b51cd4664918454011094321e464" + "sha256:4fb0c16c71ad35d278a5fa7a4106a5c26c2b2b5c5efc47c1d67635db90b6071e", + "sha256:89ae0932f4f269754790569828859eaa0ae2ce73d1f3eb1f30ae7c20d4daf5ce" ], "markers": "python_version >= '3.8'", - "version": "==23.2.1" + "version": "==24.1.0" }, "geoalchemy2": { "hashes": [ - "sha256:32069594f34473bd60830e341fac1c728b0d094c977e1c44f2b8f418f4f585b8", - "sha256:3adab4d0970710f1d4a55feda5dd2074463261150cb8d1688e3a0d73ef65881e" + "sha256:649c70f9275aded0341ed0c879e140d77c16ec33023cc21352de2eb7bb5fd509", + "sha256:e940681a60571d692124f687ecfe605164675341dbbaf5adf5f0ee46932c337b" ], "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==0.14.4" + "version": "==0.14.6" }, "geojson": { "hashes": [ @@ -121,6 +120,80 @@ "markers": "platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))", "version": "==3.0.3" }, + "jinja2": { + "hashes": [ + "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa", + "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90" + ], + "index": "pypi", + "version": "==3.1.3" + }, + "markupsafe": { + "hashes": [ + "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf", + "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff", + "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f", + "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3", + "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532", + "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f", + "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617", + "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df", + "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4", + "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906", + "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f", + "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4", + "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8", + "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371", + "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2", + "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465", + "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52", + "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6", + "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169", + "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad", + "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2", + "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0", + "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029", + "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f", + "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a", + "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced", + "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5", + "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c", + "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf", + "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9", + "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb", + "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad", + "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3", + "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1", + "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46", + "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc", + "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a", + "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee", + "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900", + "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5", + "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea", + "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f", + "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5", + "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e", + "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a", + "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f", + "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50", + "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a", + "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b", + "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4", + "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff", + "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2", + "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46", + "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b", + "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf", + "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5", + "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5", + "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab", + "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd", + "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68" + ], + "markers": "python_version >= '3.7'", + "version": "==2.1.5" + }, "numpy": { "hashes": [ "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", @@ -165,11 +238,11 @@ }, "packaging": { "hashes": [ - "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", - "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" + "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", + "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9" ], "markers": "python_version >= '3.7'", - "version": "==23.2" + "version": "==24.0" }, "pandas": { "hashes": [ @@ -286,11 +359,11 @@ }, "python-dateutil": { "hashes": [ - "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", - "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.8.2" + "version": "==2.9.0.post0" }, "pytz": { "hashes": [ @@ -356,59 +429,58 @@ }, "sqlalchemy": { "hashes": [ - "sha256:03f448ffb731b48323bda68bcc93152f751436ad6037f18a42b7e16af9e91c07", - "sha256:0de1263aac858f288a80b2071990f02082c51d88335a1db0d589237a3435fe71", - "sha256:0fb3bffc0ced37e5aa4ac2416f56d6d858f46d4da70c09bb731a246e70bff4d5", - "sha256:120af1e49d614d2525ac247f6123841589b029c318b9afbfc9e2b70e22e1827d", - "sha256:1306102f6d9e625cebaca3d4c9c8f10588735ef877f0360b5cdb4fdfd3fd7131", - "sha256:15e19a84b84528f52a68143439d0c7a3a69befcd4f50b8ef9b7b69d2628ae7c4", - "sha256:1ab4e0448018d01b142c916cc7119ca573803a4745cfe341b8f95657812700ac", - "sha256:1fc19ae2e07a067663dd24fca55f8ed06a288384f0e6e3910420bf4b1270cc51", - "sha256:2f5c9dfb0b9ab5e3a8a00249534bdd838d943ec4cfb9abe176a6c33408430230", - "sha256:30d81cc1192dc693d49d5671cd40cdec596b885b0ce3b72f323888ab1c3863d5", - "sha256:33e8bde8fff203de50399b9039c4e14e42d4d227759155c21f8da4a47fc8053c", - "sha256:4535c49d961fe9a77392e3a630a626af5baa967172d42732b7a43496c8b28876", - "sha256:48217be1de7d29a5600b5c513f3f7664b21d32e596d69582be0a94e36b8309cb", - "sha256:5ada0438f5b74c3952d916c199367c29ee4d6858edff18eab783b3978d0db16d", - "sha256:5b78aa9f4f68212248aaf8943d84c0ff0f74efc65a661c2fc68b82d498311fd5", - "sha256:5cd20f58c29bbf2680039ff9f569fa6d21453fbd2fa84dbdb4092f006424c2e6", - "sha256:611068511b5531304137bcd7fe8117c985d1b828eb86043bd944cebb7fae3910", - "sha256:680b9a36029b30cf063698755d277885d4a0eab70a2c7c6e71aab601323cba45", - "sha256:6c5bad7c60a392850d2f0fee8f355953abaec878c483dd7c3836e0089f046bf6", - "sha256:6c7a596d0be71b7baa037f4ac10d5e057d276f65a9a611c46970f012752ebf2d", - "sha256:7f470327d06400a0aa7926b375b8e8c3c31d335e0884f509fe272b3c700a7254", - "sha256:86a6ed69a71fe6b88bf9331594fa390a2adda4a49b5c06f98e47bf0d392534f8", - "sha256:8dfc936870507da96aebb43e664ae3a71a7b96278382bcfe84d277b88e379b18", - "sha256:954d9735ee9c3fa74874c830d089a815b7b48df6f6b6e357a74130e478dbd951", - "sha256:9e56afce6431450442f3ab5973156289bd5ec33dd618941283847c9fd5ff06bf", - "sha256:a3012ab65ea42de1be81fff5fb28d6db893ef978950afc8130ba707179b4284a", - "sha256:ad862295ad3f644e3c2c0d8b10a988e1600d3123ecb48702d2c0f26771f1c396", - "sha256:b1d9d1bfd96eef3c3faedb73f486c89e44e64e40e5bfec304ee163de01cf996f", - "sha256:b86abba762ecfeea359112b2bb4490802b340850bbee1948f785141a5e020de8", - "sha256:b90053be91973a6fb6020a6e44382c97739736a5a9d74e08cc29b196639eb979", - "sha256:c4fbe6a766301f2e8a4519f4500fe74ef0a8509a59e07a4085458f26228cd7cc", - "sha256:ca891af9f3289d24a490a5fde664ea04fe2f4984cd97e26de7442a4251bd4b7c", - "sha256:cb0845e934647232b6ff5150df37ceffd0b67b754b9fdbb095233deebcddbd4a", - "sha256:ce850db091bf7d2a1f2fdb615220b968aeff3849007b1204bf6e3e50a57b3d32", - "sha256:d04e579e911562f1055d26dab1868d3e0bb905db3bccf664ee8ad109f035618a", - "sha256:d07ee7793f2aeb9b80ec8ceb96bc8cc08a2aec8a1b152da1955d64e4825fcbac", - "sha256:d177b7e82f6dd5e1aebd24d9c3297c70ce09cd1d5d37b43e53f39514379c029c", - "sha256:d7b5a3e2120982b8b6bd1d5d99e3025339f7fb8b8267551c679afb39e9c7c7f1", - "sha256:d873c21b356bfaf1589b89090a4011e6532582b3a8ea568a00e0c3aab09399dd", - "sha256:d997c5938a08b5e172c30583ba6b8aad657ed9901fc24caf3a7152eeccb2f1b4", - "sha256:dbcd77c4d94b23e0753c5ed8deba8c69f331d4fd83f68bfc9db58bc8983f49cd", - "sha256:e36aa62b765cf9f43a003233a8c2d7ffdeb55bc62eaa0a0380475b228663a38f", - "sha256:e97cf143d74a7a5a0f143aa34039b4fecf11343eed66538610debc438685db4a", - "sha256:eb15ef40b833f5b2f19eeae65d65e191f039e71790dd565c2af2a3783f72262f", - "sha256:ec1f5a328464daf7a1e4e385e4f5652dd9b1d12405075ccba1df842f7774b4fc", - "sha256:f9374e270e2553653d710ece397df67db9d19c60d2647bcd35bfc616f1622dcd", - "sha256:fa67d821c1fd268a5a87922ef4940442513b4e6c377553506b9db3b83beebbd8", - "sha256:fd8aafda7cdff03b905d4426b714601c0978725a19efc39f5f207b86d188ba01", - "sha256:ff2f1b7c963961d41403b650842dc2039175b906ab2093635d8319bef0b7d620" + "sha256:0315d9125a38026227f559488fe7f7cee1bd2fbc19f9fd637739dc50bb6380b2", + "sha256:0d3dd67b5d69794cfe82862c002512683b3db038b99002171f624712fa71aeaa", + "sha256:124202b4e0edea7f08a4db8c81cc7859012f90a0d14ba2bf07c099aff6e96462", + "sha256:1ee8bd6d68578e517943f5ebff3afbd93fc65f7ef8f23becab9fa8fb315afb1d", + "sha256:243feb6882b06a2af68ecf4bec8813d99452a1b62ba2be917ce6283852cf701b", + "sha256:2858bbab1681ee5406650202950dc8f00e83b06a198741b7c656e63818633526", + "sha256:2f60843068e432311c886c5f03c4664acaef507cf716f6c60d5fde7265be9d7b", + "sha256:328529f7c7f90adcd65aed06a161851f83f475c2f664a898af574893f55d9e53", + "sha256:33157920b233bc542ce497a81a2e1452e685a11834c5763933b440fedd1d8e2d", + "sha256:3eba73ef2c30695cb7eabcdb33bb3d0b878595737479e152468f3ba97a9c22a4", + "sha256:426f2fa71331a64f5132369ede5171c52fd1df1bd9727ce621f38b5b24f48750", + "sha256:45c7b78dfc7278329f27be02c44abc0d69fe235495bb8e16ec7ef1b1a17952db", + "sha256:46a3d4e7a472bfff2d28db838669fc437964e8af8df8ee1e4548e92710929adc", + "sha256:4a5adf383c73f2d49ad15ff363a8748319ff84c371eed59ffd0127355d6ea1da", + "sha256:4b6303bfd78fb3221847723104d152e5972c22367ff66edf09120fcde5ddc2e2", + "sha256:56856b871146bfead25fbcaed098269d90b744eea5cb32a952df00d542cdd368", + "sha256:5da98815f82dce0cb31fd1e873a0cb30934971d15b74e0d78cf21f9e1b05953f", + "sha256:5df5d1dafb8eee89384fb7a1f79128118bc0ba50ce0db27a40750f6f91aa99d5", + "sha256:68722e6a550f5de2e3cfe9da6afb9a7dd15ef7032afa5651b0f0c6b3adb8815d", + "sha256:78bb7e8da0183a8301352d569900d9d3594c48ac21dc1c2ec6b3121ed8b6c986", + "sha256:81ba314a08c7ab701e621b7ad079c0c933c58cdef88593c59b90b996e8b58fa5", + "sha256:843a882cadebecc655a68bd9a5b8aa39b3c52f4a9a5572a3036fb1bb2ccdc197", + "sha256:87724e7ed2a936fdda2c05dbd99d395c91ea3c96f029a033a4a20e008dd876bf", + "sha256:8c7f10720fc34d14abad5b647bc8202202f4948498927d9f1b4df0fb1cf391b7", + "sha256:8e91b5e341f8c7f1e5020db8e5602f3ed045a29f8e27f7f565e0bdee3338f2c7", + "sha256:943aa74a11f5806ab68278284a4ddd282d3fb348a0e96db9b42cb81bf731acdc", + "sha256:9461802f2e965de5cff80c5a13bc945abea7edaa1d29360b485c3d2b56cdb075", + "sha256:9b66fcd38659cab5d29e8de5409cdf91e9986817703e1078b2fdaad731ea66f5", + "sha256:a6bec1c010a6d65b3ed88c863d56b9ea5eeefdf62b5e39cafd08c65f5ce5198b", + "sha256:a921002be69ac3ab2cf0c3017c4e6a3377f800f1fca7f254c13b5f1a2f10022c", + "sha256:aca7b6d99a4541b2ebab4494f6c8c2f947e0df4ac859ced575238e1d6ca5716b", + "sha256:ad7acbe95bac70e4e687a4dc9ae3f7a2f467aa6597049eeb6d4a662ecd990bb6", + "sha256:af8ce2d31679006e7b747d30a89cd3ac1ec304c3d4c20973f0f4ad58e2d1c4c9", + "sha256:b4a2cf92995635b64876dc141af0ef089c6eea7e05898d8d8865e71a326c0385", + "sha256:bbda76961eb8f27e6ad3c84d1dc56d5bc61ba8f02bd20fcf3450bd421c2fcc9c", + "sha256:bd7e4baf9161d076b9a7e432fce06217b9bd90cfb8f1d543d6e8c4595627edb9", + "sha256:bea30da1e76cb1acc5b72e204a920a3a7678d9d52f688f087dc08e54e2754c67", + "sha256:c61e2e41656a673b777e2f0cbbe545323dbe0d32312f590b1bc09da1de6c2a02", + "sha256:c6c4da4843e0dabde41b8f2e8147438330924114f541949e6318358a56d1875a", + "sha256:d3499008ddec83127ab286c6f6ec82a34f39c9817f020f75eca96155f9765097", + "sha256:dbb990612c36163c6072723523d2be7c3eb1517bbdd63fe50449f56afafd1133", + "sha256:dd53b6c4e6d960600fd6532b79ee28e2da489322fcf6648738134587faf767b6", + "sha256:df40c16a7e8be7413b885c9bf900d402918cc848be08a59b022478804ea076b8", + "sha256:e0a5354cb4de9b64bccb6ea33162cb83e03dbefa0d892db88a672f5aad638a75", + "sha256:e0b148ab0438f72ad21cb004ce3bdaafd28465c4276af66df3b9ecd2037bf252", + "sha256:e23b88c69497a6322b5796c0781400692eca1ae5532821b39ce81a48c395aae9", + "sha256:fc4974d3684f28b61b9a90fcb4c41fb340fd4b6a50c04365704a4da5a9603b05", + "sha256:feea693c452d85ea0015ebe3bb9cd15b6f49acc1a31c28b3c50f4db0f8fb1e71", + "sha256:fffcc8edc508801ed2e6a4e7b0d150a62196fd28b4e16ab9f65192e8186102b6" ], "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.0.27" + "version": "==2.0.28" }, "tohu": { "hashes": [ @@ -479,7 +551,7 @@ "sha256:e8a6ae970537e67830776488bca52000eaa37fa63b9988e8c487458d9cd5ace6", "sha256:faf2ee02e6612577ba0181f4347bcbcf591eb122f7841ae5ba233d12c39dcb4d" ], - "markers": "python_version >= '3.8'", + "index": "pypi", "version": "==24.2.0" }, "click": { @@ -547,11 +619,11 @@ }, "packaging": { "hashes": [ - "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", - "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" + "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", + "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9" ], "markers": "python_version >= '3.7'", - "version": "==23.2" + "version": "==24.0" }, "parso": { "hashes": [ From 7d549fd9e336d427fa77f328a3b093454a2d545f Mon Sep 17 00:00:00 2001 From: Thingus Date: Tue, 12 Mar 2024 17:00:10 +0000 Subject: [PATCH 12/31] Comments from PR (wip) POSTGRES_DB and POSTGRES_PORT aren't in development_environment; so hardcoded. Not sure if this is a good idea for POSTGRES_PORT. --- .../bin/9900_ingest_synthetic_data.sh | 2 +- flowdb/testdata/bin/9900_ingest_test_data.sh | 2 +- .../{bin => test_data/py}/run_qa_checks.py | 36 +++++++++++++------ 3 files changed, 28 insertions(+), 12 deletions(-) rename flowdb/testdata/{bin => test_data/py}/run_qa_checks.py (78%) diff --git a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh index fd10979853..9dbcdd9fe6 100644 --- a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh +++ b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh @@ -78,5 +78,5 @@ else fi if [ "${SKIP_TEST_QA_CHECK,,}" != "true" ]; then echo "Running QA checks on test data" - pipenv run python ./run_qa_checks.py --dates ${DISASTER_END:-"2015-01-01"} --event-types calls mds sms + pipenv run python ../test_data/py/run_qa_checks.py fi diff --git a/flowdb/testdata/bin/9900_ingest_test_data.sh b/flowdb/testdata/bin/9900_ingest_test_data.sh index 5f67b8953d..8d01382f8e 100644 --- a/flowdb/testdata/bin/9900_ingest_test_data.sh +++ b/flowdb/testdata/bin/9900_ingest_test_data.sh @@ -34,5 +34,5 @@ fi # &{VAR,,} should lowercase the variable on interpolation if [ "${SKIP_TEST_QA_CHECK,,}" != "true" ]; then - pipenv run python ./run_qa_checks.py --dates ${DISASTER_END:-"2015-01-01"} --event-types calls mds sms + pipenv run python ../test_data/py/run_qa_checks.py fi diff --git a/flowdb/testdata/bin/run_qa_checks.py b/flowdb/testdata/test_data/py/run_qa_checks.py similarity index 78% rename from flowdb/testdata/bin/run_qa_checks.py rename to flowdb/testdata/test_data/py/run_qa_checks.py index cc9593b8b7..bb16915d12 100644 --- a/flowdb/testdata/bin/run_qa_checks.py +++ b/flowdb/testdata/test_data/py/run_qa_checks.py @@ -9,6 +9,7 @@ from typing import List from jinja2 import Environment, PackageLoader, Template from sqlalchemy import create_engine +from sqlalchemy.engine import Engine import os import argparse @@ -62,7 +63,19 @@ def render_qa_check(template: Template, date: date, cdr_type: str) -> str: ) -if __name__ == "__main__()": +def get_available_tables(engine: Engine): + with engine.begin() as conn: + tables = conn.execute("SELECT table_name FROM available_tables") + return tables.all() + + +def get_available_dates(engine: Engine): + with engine.begin() as conn: + dates = conn.execute("SELECT cdr_date FROM etl.available_dates") + return dates.all() + + +if __name__ == "__main__": parser = argparse.ArgumentParser( description="Runs all flowetl checks for ingested data" ) @@ -70,22 +83,25 @@ def render_qa_check(template: Template, date: date, cdr_type: str) -> str: "--dates", type=lambda s: datetime.datetime.strptime(s, "%Y-%m-%d"), help="Date to run ingestion check on. Can be specified multiple times.", - nargs="+", + nargs="*", ) parser.add_argument( - "--event-types", help="Event tables to run qa checks on.", nargs="+" + "--event-types", help="Event tables to run qa checks on.", nargs="*" ) args = parser.parse_args() - db_user = os.getenv("POSTGRES_USER", "flowdb") - db_name = os.getenv("POSTGRES_DB", "flowdb") - db_port = os.getenv("POSTGRES_PORT", "9000") - db_password = os.getenv( - "POSTGRES_PASSWORD", "flowflow" - ) # In here for dev, don't think we'll need it in prod - conn_str = f"postgresql://{db_user}:{db_password}@localhost:{db_port}/{db_name}" + db_user = os.environ["POSTGRES_USER"] + db_password = os.environ["POSTGRES_PASSWORD"] + conn_str = f"postgresql://{db_user}:{db_password}@localhost:5432/flowdb" engine = create_engine(conn_str) + breakpoint() + if "dates" not in args.keys(): + args["dates"] = get_available_dates(engine) + + if "event-types" not in args.keys(): + args["event-types"] = get_available_tables(engine) + qa_scn = MockQaScenario(dates=args.dates, tables=args.tables) templates = ( From cef5b4dd35606ae2c9e22ca168b9b0a9db1017d2 Mon Sep 17 00:00:00 2001 From: Thingus Date: Wed, 13 Mar 2024 15:59:35 +0000 Subject: [PATCH 13/31] Get default events tables and dates from flowdb --- flowdb/testdata/test_data/py/run_qa_checks.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/flowdb/testdata/test_data/py/run_qa_checks.py b/flowdb/testdata/test_data/py/run_qa_checks.py index bb16915d12..1ae6166557 100644 --- a/flowdb/testdata/test_data/py/run_qa_checks.py +++ b/flowdb/testdata/test_data/py/run_qa_checks.py @@ -37,6 +37,7 @@ class QaTemplate: display_name: str template: Template + event_type: str @dataclass @@ -65,14 +66,16 @@ def render_qa_check(template: Template, date: date, cdr_type: str) -> str: def get_available_tables(engine: Engine): with engine.begin() as conn: - tables = conn.execute("SELECT table_name FROM available_tables") - return tables.all() + tables = conn.execute( + "SELECT table_name FROM available_tables WHERE has_subscribers" + ) + return [t[0] for t in tables.all()] def get_available_dates(engine: Engine): with engine.begin() as conn: dates = conn.execute("SELECT cdr_date FROM etl.available_dates") - return dates.all() + return [d[0] for d in dates.all()] if __name__ == "__main__": @@ -86,26 +89,30 @@ def get_available_dates(engine: Engine): nargs="*", ) parser.add_argument( - "--event-types", help="Event tables to run qa checks on.", nargs="*" + "--event_types", help="Event tables to run qa checks on.", nargs="*" ) args = parser.parse_args() db_user = os.environ["POSTGRES_USER"] db_password = os.environ["POSTGRES_PASSWORD"] - conn_str = f"postgresql://{db_user}:{db_password}@localhost:5432/flowdb" + db_port = os.getenv("POSTGRES_PORT", 5432) # For local running + conn_str = f"postgresql://{db_user}:{db_password}@localhost:{db_port}/flowdb" engine = create_engine(conn_str) - breakpoint() - if "dates" not in args.keys(): - args["dates"] = get_available_dates(engine) + dates = get_available_dates(engine) if not args.dates else args.dates - if "event-types" not in args.keys(): - args["event-types"] = get_available_tables(engine) + event_types = ( + get_available_tables(engine) if not args.event_types else args.event_types + ) - qa_scn = MockQaScenario(dates=args.dates, tables=args.tables) + qa_scn = MockQaScenario(dates=dates, tables=event_types) templates = ( - QaTemplate(Path(t).name, env.get_template(t)) + QaTemplate( + Path(t).name, + env.get_template(t), + Path(t).parent if Path(t).parent != Path(".") else "any", + ) for t in env.list_templates(".sql") ) @@ -119,6 +126,7 @@ def get_available_dates(engine: Engine): datetime.now(), ) for date, cdr_type, template in product(qa_scn.dates, qa_scn.tables, templates) + if template.event_type in [cdr_type, "any"] ) with engine.begin() as conn: From a23b9f00495a93e4194585cdf1a5c18040b5e2a9 Mon Sep 17 00:00:00 2001 From: Thingus Date: Thu, 14 Mar 2024 10:17:35 +0000 Subject: [PATCH 14/31] Moving run_qa_checks to bin + adding to dockerignoreignore --- flowdb/testdata/bin/9900_ingest_synthetic_data.sh | 2 +- flowdb/testdata/bin/9900_ingest_test_data.sh | 2 +- flowdb/testdata/{test_data/py => bin}/run_qa_checks.py | 0 flowdb_synthetic_data.Dockerfile.dockerignore | 2 ++ flowdb_testdata.Dockerfile.dockerignore | 3 ++- 5 files changed, 6 insertions(+), 3 deletions(-) rename flowdb/testdata/{test_data/py => bin}/run_qa_checks.py (100%) diff --git a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh index 9dbcdd9fe6..89b52bcb12 100644 --- a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh +++ b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh @@ -78,5 +78,5 @@ else fi if [ "${SKIP_TEST_QA_CHECK,,}" != "true" ]; then echo "Running QA checks on test data" - pipenv run python ../test_data/py/run_qa_checks.py + pipenv run python run_qa_checks.py fi diff --git a/flowdb/testdata/bin/9900_ingest_test_data.sh b/flowdb/testdata/bin/9900_ingest_test_data.sh index 8d01382f8e..8f5ce530b4 100644 --- a/flowdb/testdata/bin/9900_ingest_test_data.sh +++ b/flowdb/testdata/bin/9900_ingest_test_data.sh @@ -34,5 +34,5 @@ fi # &{VAR,,} should lowercase the variable on interpolation if [ "${SKIP_TEST_QA_CHECK,,}" != "true" ]; then - pipenv run python ../test_data/py/run_qa_checks.py + pipenv run python run_qa_checks.py fi diff --git a/flowdb/testdata/test_data/py/run_qa_checks.py b/flowdb/testdata/bin/run_qa_checks.py similarity index 100% rename from flowdb/testdata/test_data/py/run_qa_checks.py rename to flowdb/testdata/bin/run_qa_checks.py diff --git a/flowdb_synthetic_data.Dockerfile.dockerignore b/flowdb_synthetic_data.Dockerfile.dockerignore index 0ed8d196c7..015686a39f 100644 --- a/flowdb_synthetic_data.Dockerfile.dockerignore +++ b/flowdb_synthetic_data.Dockerfile.dockerignore @@ -16,4 +16,6 @@ !flowdb/testdata/synthetic_data/data/NPL_admbnda_adm3_Districts_simplified.geojson !flowdb/testdata/synthetic_data/Pipfile* !flowdb/testdata/bin/9910_run_synthetic_dfs_data_generation_script.sh +!flowdb/testdata/bin/run_qa_checks.py !flowdb/testdata/test_data/py/ + diff --git a/flowdb_testdata.Dockerfile.dockerignore b/flowdb_testdata.Dockerfile.dockerignore index fa6b71a5cc..87de5db736 100644 --- a/flowdb_testdata.Dockerfile.dockerignore +++ b/flowdb_testdata.Dockerfile.dockerignore @@ -13,8 +13,9 @@ !flowdb/testdata/bin/9900_ingest_test_data.sh !flowdb/testdata/bin/9910_run_synthetic_dfs_data_generation_script.sh !flowdb/testdata/bin/9910_migrate_test_data.sql +!flowdb/testdata/bin/run_qa_checks.py !flowdb/testdata/test_data/Pipfile !flowdb/testdata/test_data/Pipfile.lock !flowdb/testdata/test_data/sql/ !flowdb/testdata/test_data/py/ -!flowdb/testdata/test_data/data/ \ No newline at end of file +!flowdb/testdata/test_data/data/ From bf1d1e2c610e3e8ca0f0707b44a078a62c02374b Mon Sep 17 00:00:00 2001 From: Thingus Date: Thu, 14 Mar 2024 10:48:27 +0000 Subject: [PATCH 15/31] Update flowdb/testdata/bin/run_qa_checks.py Co-authored-by: Jonathan Gray --- flowdb/testdata/bin/run_qa_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowdb/testdata/bin/run_qa_checks.py b/flowdb/testdata/bin/run_qa_checks.py index 1ae6166557..97e3047bc2 100644 --- a/flowdb/testdata/bin/run_qa_checks.py +++ b/flowdb/testdata/bin/run_qa_checks.py @@ -58,7 +58,7 @@ class MockQaScenario: def render_qa_check(template: Template, date: date, cdr_type: str) -> str: return template.render( - final_table=f"events.{cdr_type}", + final_table=f"events.{cdr_type}_{date.strftime('%Y%m%d')", cdr_type=cdr_type, ds=date.strftime("%Y-%m-%d"), ) From 91b78ae7ecf2eef8ba756ccf6e64ab2a2d062135 Mon Sep 17 00:00:00 2001 From: Thingus Date: Thu, 14 Mar 2024 10:59:56 +0000 Subject: [PATCH 16/31] Jinja2 and relocking for synthdata --- flowdb/testdata/synthetic_data/Pipfile | 2 +- flowdb/testdata/synthetic_data/Pipfile.lock | 207 +++++++++++++------- 2 files changed, 139 insertions(+), 70 deletions(-) diff --git a/flowdb/testdata/synthetic_data/Pipfile b/flowdb/testdata/synthetic_data/Pipfile index 9525e7e23b..c2ac7a65ab 100644 --- a/flowdb/testdata/synthetic_data/Pipfile +++ b/flowdb/testdata/synthetic_data/Pipfile @@ -11,7 +11,7 @@ structlog = "*" tohu = "==0.6.7" numpy = "<=1.26.2" # Tohu uses float division where it should be using int division, and hence passes a float where numpy expects an int (https://github.com/maxalbert/tohu/blob/3adf0c58b13ef1e1d716d7d613484d2adc58fb60/tohu/v6/primitive_generators.py#L335) # This used to work, but doesn't as of numpy 1.26.0 (although I haven't managed to track down the relevant change or find a corresponding issue or changelog entry) - +jinja2 = "*" [dev-packages] black = {extras = ["jupyter"],version = "==24.2.0"} diff --git a/flowdb/testdata/synthetic_data/Pipfile.lock b/flowdb/testdata/synthetic_data/Pipfile.lock index 3b484c963d..845108541c 100644 --- a/flowdb/testdata/synthetic_data/Pipfile.lock +++ b/flowdb/testdata/synthetic_data/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "becdb6611f2dd33f1561dd87947eb801fe447b30a50a77270cc03a58c31e3b25" + "sha256": "34419b6c96ff4c58470acf34a32dd94f779b5bbf39f0d7a1247a2ab6f0ad5ee6" }, "pipfile-spec": 6, "requires": { @@ -34,11 +34,11 @@ }, "faker": { "hashes": [ - "sha256:0520a6b97e07c658b2798d7140971c1d5bc4bcd5013e7937fe075fd054aa320c", - "sha256:f07b64d27f67b62c7f0536a72f47813015b3b51cd4664918454011094321e464" + "sha256:87d5e7730426e7b36817921679c4eaf3d810cedb8c81194f47adc3df2122ca18", + "sha256:dce4754921f9fa7e2003c26834093361b8f45072e0f46f172d6ca1234774ecd4" ], "markers": "python_version >= '3.8'", - "version": "==23.2.1" + "version": "==24.2.0" }, "geojson": { "hashes": [ @@ -112,6 +112,80 @@ "markers": "platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))", "version": "==3.0.3" }, + "jinja2": { + "hashes": [ + "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa", + "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90" + ], + "index": "pypi", + "version": "==3.1.3" + }, + "markupsafe": { + "hashes": [ + "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf", + "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff", + "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f", + "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3", + "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532", + "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f", + "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617", + "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df", + "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4", + "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906", + "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f", + "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4", + "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8", + "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371", + "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2", + "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465", + "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52", + "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6", + "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169", + "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad", + "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2", + "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0", + "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029", + "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f", + "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a", + "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced", + "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5", + "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c", + "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf", + "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9", + "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb", + "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad", + "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3", + "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1", + "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46", + "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc", + "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a", + "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee", + "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900", + "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5", + "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea", + "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f", + "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5", + "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e", + "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a", + "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f", + "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50", + "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a", + "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b", + "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4", + "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff", + "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2", + "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46", + "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b", + "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf", + "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5", + "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5", + "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab", + "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd", + "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68" + ], + "markers": "python_version >= '3.7'", + "version": "==2.1.5" + }, "numpy": { "hashes": [ "sha256:06fa1ed84aa60ea6ef9f91ba57b5ed963c3729534e6e54055fc151fad0423f0a", @@ -152,7 +226,6 @@ "sha256:fe6b44fb8fcdf7eda4ef4461b97b3f63c466b27ab151bec2366db8b197387841" ], "index": "pypi", - "markers": "python_version >= '3.9'", "version": "==1.26.2" }, "pandas": { @@ -188,7 +261,6 @@ "sha256:f9d3558d263073ed95e46f4650becff0c5e1ffe0fc3a015de3c79283dfbdb3df" ], "index": "pypi", - "markers": "python_version >= '3.9'", "version": "==2.2.1" }, "psycopg2-binary": { @@ -267,16 +339,15 @@ "sha256:f9b5571d33660d5009a8b3c25dc1db560206e2d2f89d3df1cb32d72c0d117d52" ], "index": "pypi", - "markers": "python_version >= '3.7'", "version": "==2.9.9" }, "python-dateutil": { "hashes": [ - "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", - "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.8.2" + "version": "==2.9.0.post0" }, "pytz": { "hashes": [ @@ -342,59 +413,58 @@ }, "sqlalchemy": { "hashes": [ - "sha256:03f448ffb731b48323bda68bcc93152f751436ad6037f18a42b7e16af9e91c07", - "sha256:0de1263aac858f288a80b2071990f02082c51d88335a1db0d589237a3435fe71", - "sha256:0fb3bffc0ced37e5aa4ac2416f56d6d858f46d4da70c09bb731a246e70bff4d5", - "sha256:120af1e49d614d2525ac247f6123841589b029c318b9afbfc9e2b70e22e1827d", - "sha256:1306102f6d9e625cebaca3d4c9c8f10588735ef877f0360b5cdb4fdfd3fd7131", - "sha256:15e19a84b84528f52a68143439d0c7a3a69befcd4f50b8ef9b7b69d2628ae7c4", - "sha256:1ab4e0448018d01b142c916cc7119ca573803a4745cfe341b8f95657812700ac", - "sha256:1fc19ae2e07a067663dd24fca55f8ed06a288384f0e6e3910420bf4b1270cc51", - "sha256:2f5c9dfb0b9ab5e3a8a00249534bdd838d943ec4cfb9abe176a6c33408430230", - "sha256:30d81cc1192dc693d49d5671cd40cdec596b885b0ce3b72f323888ab1c3863d5", - "sha256:33e8bde8fff203de50399b9039c4e14e42d4d227759155c21f8da4a47fc8053c", - "sha256:4535c49d961fe9a77392e3a630a626af5baa967172d42732b7a43496c8b28876", - "sha256:48217be1de7d29a5600b5c513f3f7664b21d32e596d69582be0a94e36b8309cb", - "sha256:5ada0438f5b74c3952d916c199367c29ee4d6858edff18eab783b3978d0db16d", - "sha256:5b78aa9f4f68212248aaf8943d84c0ff0f74efc65a661c2fc68b82d498311fd5", - "sha256:5cd20f58c29bbf2680039ff9f569fa6d21453fbd2fa84dbdb4092f006424c2e6", - "sha256:611068511b5531304137bcd7fe8117c985d1b828eb86043bd944cebb7fae3910", - "sha256:680b9a36029b30cf063698755d277885d4a0eab70a2c7c6e71aab601323cba45", - "sha256:6c5bad7c60a392850d2f0fee8f355953abaec878c483dd7c3836e0089f046bf6", - "sha256:6c7a596d0be71b7baa037f4ac10d5e057d276f65a9a611c46970f012752ebf2d", - "sha256:7f470327d06400a0aa7926b375b8e8c3c31d335e0884f509fe272b3c700a7254", - "sha256:86a6ed69a71fe6b88bf9331594fa390a2adda4a49b5c06f98e47bf0d392534f8", - "sha256:8dfc936870507da96aebb43e664ae3a71a7b96278382bcfe84d277b88e379b18", - "sha256:954d9735ee9c3fa74874c830d089a815b7b48df6f6b6e357a74130e478dbd951", - "sha256:9e56afce6431450442f3ab5973156289bd5ec33dd618941283847c9fd5ff06bf", - "sha256:a3012ab65ea42de1be81fff5fb28d6db893ef978950afc8130ba707179b4284a", - "sha256:ad862295ad3f644e3c2c0d8b10a988e1600d3123ecb48702d2c0f26771f1c396", - "sha256:b1d9d1bfd96eef3c3faedb73f486c89e44e64e40e5bfec304ee163de01cf996f", - "sha256:b86abba762ecfeea359112b2bb4490802b340850bbee1948f785141a5e020de8", - "sha256:b90053be91973a6fb6020a6e44382c97739736a5a9d74e08cc29b196639eb979", - "sha256:c4fbe6a766301f2e8a4519f4500fe74ef0a8509a59e07a4085458f26228cd7cc", - "sha256:ca891af9f3289d24a490a5fde664ea04fe2f4984cd97e26de7442a4251bd4b7c", - "sha256:cb0845e934647232b6ff5150df37ceffd0b67b754b9fdbb095233deebcddbd4a", - "sha256:ce850db091bf7d2a1f2fdb615220b968aeff3849007b1204bf6e3e50a57b3d32", - "sha256:d04e579e911562f1055d26dab1868d3e0bb905db3bccf664ee8ad109f035618a", - "sha256:d07ee7793f2aeb9b80ec8ceb96bc8cc08a2aec8a1b152da1955d64e4825fcbac", - "sha256:d177b7e82f6dd5e1aebd24d9c3297c70ce09cd1d5d37b43e53f39514379c029c", - "sha256:d7b5a3e2120982b8b6bd1d5d99e3025339f7fb8b8267551c679afb39e9c7c7f1", - "sha256:d873c21b356bfaf1589b89090a4011e6532582b3a8ea568a00e0c3aab09399dd", - "sha256:d997c5938a08b5e172c30583ba6b8aad657ed9901fc24caf3a7152eeccb2f1b4", - "sha256:dbcd77c4d94b23e0753c5ed8deba8c69f331d4fd83f68bfc9db58bc8983f49cd", - "sha256:e36aa62b765cf9f43a003233a8c2d7ffdeb55bc62eaa0a0380475b228663a38f", - "sha256:e97cf143d74a7a5a0f143aa34039b4fecf11343eed66538610debc438685db4a", - "sha256:eb15ef40b833f5b2f19eeae65d65e191f039e71790dd565c2af2a3783f72262f", - "sha256:ec1f5a328464daf7a1e4e385e4f5652dd9b1d12405075ccba1df842f7774b4fc", - "sha256:f9374e270e2553653d710ece397df67db9d19c60d2647bcd35bfc616f1622dcd", - "sha256:fa67d821c1fd268a5a87922ef4940442513b4e6c377553506b9db3b83beebbd8", - "sha256:fd8aafda7cdff03b905d4426b714601c0978725a19efc39f5f207b86d188ba01", - "sha256:ff2f1b7c963961d41403b650842dc2039175b906ab2093635d8319bef0b7d620" + "sha256:0315d9125a38026227f559488fe7f7cee1bd2fbc19f9fd637739dc50bb6380b2", + "sha256:0d3dd67b5d69794cfe82862c002512683b3db038b99002171f624712fa71aeaa", + "sha256:124202b4e0edea7f08a4db8c81cc7859012f90a0d14ba2bf07c099aff6e96462", + "sha256:1ee8bd6d68578e517943f5ebff3afbd93fc65f7ef8f23becab9fa8fb315afb1d", + "sha256:243feb6882b06a2af68ecf4bec8813d99452a1b62ba2be917ce6283852cf701b", + "sha256:2858bbab1681ee5406650202950dc8f00e83b06a198741b7c656e63818633526", + "sha256:2f60843068e432311c886c5f03c4664acaef507cf716f6c60d5fde7265be9d7b", + "sha256:328529f7c7f90adcd65aed06a161851f83f475c2f664a898af574893f55d9e53", + "sha256:33157920b233bc542ce497a81a2e1452e685a11834c5763933b440fedd1d8e2d", + "sha256:3eba73ef2c30695cb7eabcdb33bb3d0b878595737479e152468f3ba97a9c22a4", + "sha256:426f2fa71331a64f5132369ede5171c52fd1df1bd9727ce621f38b5b24f48750", + "sha256:45c7b78dfc7278329f27be02c44abc0d69fe235495bb8e16ec7ef1b1a17952db", + "sha256:46a3d4e7a472bfff2d28db838669fc437964e8af8df8ee1e4548e92710929adc", + "sha256:4a5adf383c73f2d49ad15ff363a8748319ff84c371eed59ffd0127355d6ea1da", + "sha256:4b6303bfd78fb3221847723104d152e5972c22367ff66edf09120fcde5ddc2e2", + "sha256:56856b871146bfead25fbcaed098269d90b744eea5cb32a952df00d542cdd368", + "sha256:5da98815f82dce0cb31fd1e873a0cb30934971d15b74e0d78cf21f9e1b05953f", + "sha256:5df5d1dafb8eee89384fb7a1f79128118bc0ba50ce0db27a40750f6f91aa99d5", + "sha256:68722e6a550f5de2e3cfe9da6afb9a7dd15ef7032afa5651b0f0c6b3adb8815d", + "sha256:78bb7e8da0183a8301352d569900d9d3594c48ac21dc1c2ec6b3121ed8b6c986", + "sha256:81ba314a08c7ab701e621b7ad079c0c933c58cdef88593c59b90b996e8b58fa5", + "sha256:843a882cadebecc655a68bd9a5b8aa39b3c52f4a9a5572a3036fb1bb2ccdc197", + "sha256:87724e7ed2a936fdda2c05dbd99d395c91ea3c96f029a033a4a20e008dd876bf", + "sha256:8c7f10720fc34d14abad5b647bc8202202f4948498927d9f1b4df0fb1cf391b7", + "sha256:8e91b5e341f8c7f1e5020db8e5602f3ed045a29f8e27f7f565e0bdee3338f2c7", + "sha256:943aa74a11f5806ab68278284a4ddd282d3fb348a0e96db9b42cb81bf731acdc", + "sha256:9461802f2e965de5cff80c5a13bc945abea7edaa1d29360b485c3d2b56cdb075", + "sha256:9b66fcd38659cab5d29e8de5409cdf91e9986817703e1078b2fdaad731ea66f5", + "sha256:a6bec1c010a6d65b3ed88c863d56b9ea5eeefdf62b5e39cafd08c65f5ce5198b", + "sha256:a921002be69ac3ab2cf0c3017c4e6a3377f800f1fca7f254c13b5f1a2f10022c", + "sha256:aca7b6d99a4541b2ebab4494f6c8c2f947e0df4ac859ced575238e1d6ca5716b", + "sha256:ad7acbe95bac70e4e687a4dc9ae3f7a2f467aa6597049eeb6d4a662ecd990bb6", + "sha256:af8ce2d31679006e7b747d30a89cd3ac1ec304c3d4c20973f0f4ad58e2d1c4c9", + "sha256:b4a2cf92995635b64876dc141af0ef089c6eea7e05898d8d8865e71a326c0385", + "sha256:bbda76961eb8f27e6ad3c84d1dc56d5bc61ba8f02bd20fcf3450bd421c2fcc9c", + "sha256:bd7e4baf9161d076b9a7e432fce06217b9bd90cfb8f1d543d6e8c4595627edb9", + "sha256:bea30da1e76cb1acc5b72e204a920a3a7678d9d52f688f087dc08e54e2754c67", + "sha256:c61e2e41656a673b777e2f0cbbe545323dbe0d32312f590b1bc09da1de6c2a02", + "sha256:c6c4da4843e0dabde41b8f2e8147438330924114f541949e6318358a56d1875a", + "sha256:d3499008ddec83127ab286c6f6ec82a34f39c9817f020f75eca96155f9765097", + "sha256:dbb990612c36163c6072723523d2be7c3eb1517bbdd63fe50449f56afafd1133", + "sha256:dd53b6c4e6d960600fd6532b79ee28e2da489322fcf6648738134587faf767b6", + "sha256:df40c16a7e8be7413b885c9bf900d402918cc848be08a59b022478804ea076b8", + "sha256:e0a5354cb4de9b64bccb6ea33162cb83e03dbefa0d892db88a672f5aad638a75", + "sha256:e0b148ab0438f72ad21cb004ce3bdaafd28465c4276af66df3b9ecd2037bf252", + "sha256:e23b88c69497a6322b5796c0781400692eca1ae5532821b39ce81a48c395aae9", + "sha256:fc4974d3684f28b61b9a90fcb4c41fb340fd4b6a50c04365704a4da5a9603b05", + "sha256:feea693c452d85ea0015ebe3bb9cd15b6f49acc1a31c28b3c50f4db0f8fb1e71", + "sha256:fffcc8edc508801ed2e6a4e7b0d150a62196fd28b4e16ab9f65192e8186102b6" ], "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.0.27" + "version": "==2.0.28" }, "structlog": { "hashes": [ @@ -402,7 +472,6 @@ "sha256:41a09886e4d55df25bdcb9b5c9674bccfab723ff43e0a86a1b7b236be8e57b16" ], "index": "pypi", - "markers": "python_version >= '3.8'", "version": "==24.1.0" }, "tohu": { @@ -474,7 +543,7 @@ "sha256:e8a6ae970537e67830776488bca52000eaa37fa63b9988e8c487458d9cd5ace6", "sha256:faf2ee02e6612577ba0181f4347bcbcf591eb122f7841ae5ba233d12c39dcb4d" ], - "markers": "python_version >= '3.8'", + "index": "pypi", "version": "==24.2.0" }, "click": { @@ -542,11 +611,11 @@ }, "packaging": { "hashes": [ - "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", - "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" + "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", + "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9" ], "markers": "python_version >= '3.7'", - "version": "==23.2" + "version": "==24.0" }, "parso": { "hashes": [ @@ -642,11 +711,11 @@ }, "traitlets": { "hashes": [ - "sha256:2e5a030e6eff91737c643231bfcf04a65b0132078dad75e4936700b213652e74", - "sha256:8585105b371a04b8316a43d5ce29c098575c2e477850b62b848b964f1444527e" + "sha256:8cdd83c040dab7d1dee822678e5f5d100b514f7b72b01615b26fc5718916fdf9", + "sha256:fcdf85684a772ddeba87db2f398ce00b40ff550d1528c03c14dbf6a02003cd80" ], "markers": "python_version >= '3.8'", - "version": "==5.14.1" + "version": "==5.14.2" }, "typing-extensions": { "hashes": [ From 46eb93ea46646ee1a240e7d398e40f556a737ec1 Mon Sep 17 00:00:00 2001 From: Thingus Date: Thu, 14 Mar 2024 11:08:41 +0000 Subject: [PATCH 17/31] Adding flowetl qa templates to flowdb test data containers --- flowdb_synthetic_data.Dockerfile.dockerignore | 2 +- flowdb_testdata.Dockerfile.dockerignore | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/flowdb_synthetic_data.Dockerfile.dockerignore b/flowdb_synthetic_data.Dockerfile.dockerignore index 015686a39f..8361533494 100644 --- a/flowdb_synthetic_data.Dockerfile.dockerignore +++ b/flowdb_synthetic_data.Dockerfile.dockerignore @@ -18,4 +18,4 @@ !flowdb/testdata/bin/9910_run_synthetic_dfs_data_generation_script.sh !flowdb/testdata/bin/run_qa_checks.py !flowdb/testdata/test_data/py/ - +!flowetl/flowetl/flowetl/qa_checks/qa_checks diff --git a/flowdb_testdata.Dockerfile.dockerignore b/flowdb_testdata.Dockerfile.dockerignore index 87de5db736..3d51b8701e 100644 --- a/flowdb_testdata.Dockerfile.dockerignore +++ b/flowdb_testdata.Dockerfile.dockerignore @@ -19,3 +19,5 @@ !flowdb/testdata/test_data/sql/ !flowdb/testdata/test_data/py/ !flowdb/testdata/test_data/data/ +!flowetl/flowetl/flowetl/qa_checks/qa_checks + From 470335da2f9b5db6a2f709bac0bd8b1f516611b1 Mon Sep 17 00:00:00 2001 From: Thingus Date: Thu, 14 Mar 2024 12:15:19 +0000 Subject: [PATCH 18/31] Maybe its those commas causing trouble --- flowdb/testdata/bin/9900_ingest_synthetic_data.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh index 89b52bcb12..2174bc6076 100644 --- a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh +++ b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh @@ -76,7 +76,7 @@ else echo "Must set SYNTHETIC_DATA_GENERATOR environment variable to 'sql' or 'python'." exit 1 fi -if [ "${SKIP_TEST_QA_CHECK,,}" != "true" ]; then +if [ "${SKIP_TEST_QA_CHECK}" != "true" ]; then echo "Running QA checks on test data" pipenv run python run_qa_checks.py fi From ae01aafd72b9eebdb4d249900a496d2f88cd5fcf Mon Sep 17 00:00:00 2001 From: Thingus Date: Thu, 14 Mar 2024 16:50:53 +0000 Subject: [PATCH 19/31] Now using pop and pushdir in test_data scripts --- flowdb/testdata/bin/9900_ingest_synthetic_data.sh | 3 ++- flowdb/testdata/bin/9900_ingest_test_data.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh index 2174bc6076..3734a71946 100644 --- a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh +++ b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh @@ -23,7 +23,7 @@ fi if [ $count != 0 ]; then echo "Found $count SQL data files in directory." if [ "$(ls -A $DIR)" ]; then - cd $DIR + pushdir $DIR for f in *.sql do echo "Running" @@ -31,6 +31,7 @@ if [ $count != 0 ]; then psql --dbname="$POSTGRES_DB" -f $f echo "------------- // Done // ---------------" done + popdir else echo "$DIR is empty." fi diff --git a/flowdb/testdata/bin/9900_ingest_test_data.sh b/flowdb/testdata/bin/9900_ingest_test_data.sh index 8f5ce530b4..5a1a82a0ae 100644 --- a/flowdb/testdata/bin/9900_ingest_test_data.sh +++ b/flowdb/testdata/bin/9900_ingest_test_data.sh @@ -19,7 +19,7 @@ fi if [ $count != 0 ]; then echo "Found $count SQL data files in directory." if [ "$(ls -A $DIR)" ]; then - cd $DIR + pushdir $DIR for f in *.sql do echo "Running" @@ -27,6 +27,7 @@ if [ $count != 0 ]; then psql --dbname="$POSTGRES_DB" -f $f echo "------------- // Done // ---------------" done + popdir else echo "$DIR is empty." fi From 6f074b54df22e9a076b42e5e59722a24d82009a3 Mon Sep 17 00:00:00 2001 From: Thingus Date: Fri, 15 Mar 2024 09:44:47 +0000 Subject: [PATCH 20/31] Some container messing --- flowdb/testdata/bin/9900_ingest_synthetic_data.sh | 2 +- flowdb/testdata/bin/9900_ingest_test_data.sh | 3 ++- flowdb/testdata/bin/run_qa_checks.py | 9 ++++----- flowdb_synthetic_data.Dockerfile | 4 ++++ flowdb_testdata.Dockerfile | 1 + 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh index 3734a71946..13a9cb56f6 100644 --- a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh +++ b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh @@ -79,5 +79,5 @@ else fi if [ "${SKIP_TEST_QA_CHECK}" != "true" ]; then echo "Running QA checks on test data" - pipenv run python run_qa_checks.py + pipenv run python run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks fi diff --git a/flowdb/testdata/bin/9900_ingest_test_data.sh b/flowdb/testdata/bin/9900_ingest_test_data.sh index 5a1a82a0ae..c33f407213 100644 --- a/flowdb/testdata/bin/9900_ingest_test_data.sh +++ b/flowdb/testdata/bin/9900_ingest_test_data.sh @@ -35,5 +35,6 @@ fi # &{VAR,,} should lowercase the variable on interpolation if [ "${SKIP_TEST_QA_CHECK,,}" != "true" ]; then - pipenv run python run_qa_checks.py + echo "Running qa checks in /docker-entrypoint-initdb.d/qa_checks" + pipenv run python run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks fi diff --git a/flowdb/testdata/bin/run_qa_checks.py b/flowdb/testdata/bin/run_qa_checks.py index 97e3047bc2..c7addcae9e 100644 --- a/flowdb/testdata/bin/run_qa_checks.py +++ b/flowdb/testdata/bin/run_qa_checks.py @@ -14,8 +14,6 @@ import argparse -env = Environment(loader=PackageLoader("flowetl", "qa_checks/qa_checks")) - update_template_string = """ INSERT INTO etl.post_etl_queries (cdr_date, cdr_type, type_of_query_or_check, outcome, optional_comment_or_description, timestamp) @@ -30,8 +28,6 @@ """ -update_template = env.from_string(update_template_string) - @dataclass class QaTemplate: @@ -58,7 +54,7 @@ class MockQaScenario: def render_qa_check(template: Template, date: date, cdr_type: str) -> str: return template.render( - final_table=f"events.{cdr_type}_{date.strftime('%Y%m%d')", + final_table=f"events.{cdr_type}_{date.strftime('%Y%m%d')}", cdr_type=cdr_type, ds=date.strftime("%Y-%m-%d"), ) @@ -82,6 +78,7 @@ def get_available_dates(engine: Engine): parser = argparse.ArgumentParser( description="Runs all flowetl checks for ingested data" ) + parser.add_argument("template_path", help="Path to the QA templates") parser.add_argument( "--dates", type=lambda s: datetime.datetime.strptime(s, "%Y-%m-%d"), @@ -92,6 +89,8 @@ def get_available_dates(engine: Engine): "--event_types", help="Event tables to run qa checks on.", nargs="*" ) args = parser.parse_args() + env = Environment(loader=PathLoader(args.qa_template_path)) + update_template = env.from_string(update_template_string) db_user = os.environ["POSTGRES_USER"] db_password = os.environ["POSTGRES_PASSWORD"] diff --git a/flowdb_synthetic_data.Dockerfile b/flowdb_synthetic_data.Dockerfile index 73e7ee0d1b..82f67db113 100644 --- a/flowdb_synthetic_data.Dockerfile +++ b/flowdb_synthetic_data.Dockerfile @@ -44,6 +44,10 @@ COPY --chown=postgres flowdb/testdata/test_data/py/* /docker-entrypoint-initdb.d COPY --chown=postgres flowdb/testdata/bin/generate_synthetic_data*.py /opt/synthetic_data/ ADD --chown=postgres flowdb/testdata/test_data/sql/admin*.sql /docker-entrypoint-initdb.d/sql/syntheticdata/ ADD --chown=postgres flowdb/testdata/synthetic_data/data/NPL_admbnda_adm3_Districts_simplified.geojson /opt/synthetic_data/ +# Copy QA templates from flowetl + +COPY --chown=postgres flowetl/flowetl/flowetl/qa_checks/qa_checks /docker-entrypoint-initdb.d/qa_checks + # Need to make postgres is owner of any subdirectrories RUN mkdir docker-entrypoint-initdb.d/sql/syntheticdata/sql && chown -R postgres /docker-entrypoint-initdb.d # Need to relax the permissions in case the container is running as an arbitrary user with a bind mount diff --git a/flowdb_testdata.Dockerfile b/flowdb_testdata.Dockerfile index 439ad1e4de..7a2f8269dd 100644 --- a/flowdb_testdata.Dockerfile +++ b/flowdb_testdata.Dockerfile @@ -41,6 +41,7 @@ ADD --chown=postgres flowdb/testdata/test_data/sql/* /docker-entrypoint-initdb.d ADD --chown=postgres flowdb/testdata/test_data/py/* /docker-entrypoint-initdb.d/py/testdata/ ADD --chown=postgres flowdb/testdata/test_data/data/ /docker-entrypoint-initdb.d/data/ COPY --chown=postgres flowdb/testdata/test_data/data/*.csv /docker-entrypoint-initdb.d/data/csv/ +COPY --chown=postgres flowetl/flowetl/flowetl/qa_checks/qa_checks /docker-entrypoint-initdb.d/qa_checks # Need to make postgres owner of any subdirectories RUN chown -R postgres /docker-entrypoint-initdb.d # Explicitly set number of days of dfs data to match test data From 8349130bce6c62cc4683c89cc6aa0b922d1fd459 Mon Sep 17 00:00:00 2001 From: Thingus Date: Fri, 15 Mar 2024 10:12:00 +0000 Subject: [PATCH 21/31] pushd not pushdir --- flowdb/testdata/bin/9900_ingest_synthetic_data.sh | 4 ++-- flowdb/testdata/bin/9900_ingest_test_data.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh index 13a9cb56f6..2b69841112 100644 --- a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh +++ b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh @@ -23,7 +23,7 @@ fi if [ $count != 0 ]; then echo "Found $count SQL data files in directory." if [ "$(ls -A $DIR)" ]; then - pushdir $DIR + pushd $DIR for f in *.sql do echo "Running" @@ -31,7 +31,7 @@ if [ $count != 0 ]; then psql --dbname="$POSTGRES_DB" -f $f echo "------------- // Done // ---------------" done - popdir + popd else echo "$DIR is empty." fi diff --git a/flowdb/testdata/bin/9900_ingest_test_data.sh b/flowdb/testdata/bin/9900_ingest_test_data.sh index c33f407213..1beac1cb5e 100644 --- a/flowdb/testdata/bin/9900_ingest_test_data.sh +++ b/flowdb/testdata/bin/9900_ingest_test_data.sh @@ -19,7 +19,7 @@ fi if [ $count != 0 ]; then echo "Found $count SQL data files in directory." if [ "$(ls -A $DIR)" ]; then - pushdir $DIR + pushd $DIR for f in *.sql do echo "Running" @@ -27,7 +27,7 @@ if [ $count != 0 ]; then psql --dbname="$POSTGRES_DB" -f $f echo "------------- // Done // ---------------" done - popdir + popd else echo "$DIR is empty." fi From 9f4b8faa04df2a03a2974de5812555892c4320ca Mon Sep 17 00:00:00 2001 From: Thingus Date: Fri, 15 Mar 2024 10:55:01 +0000 Subject: [PATCH 22/31] Pipenv messing --- flowdb/testdata/bin/9900_ingest_synthetic_data.sh | 5 ++++- flowdb/testdata/bin/9900_ingest_test_data.sh | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh index 2b69841112..8632e1b87c 100644 --- a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh +++ b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh @@ -79,5 +79,8 @@ else fi if [ "${SKIP_TEST_QA_CHECK}" != "true" ]; then echo "Running QA checks on test data" - pipenv run python run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks + pushd ../synthetic_data + pipenv shell + popd + python run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks fi diff --git a/flowdb/testdata/bin/9900_ingest_test_data.sh b/flowdb/testdata/bin/9900_ingest_test_data.sh index 1beac1cb5e..7c01c22ff9 100644 --- a/flowdb/testdata/bin/9900_ingest_test_data.sh +++ b/flowdb/testdata/bin/9900_ingest_test_data.sh @@ -36,5 +36,8 @@ fi # &{VAR,,} should lowercase the variable on interpolation if [ "${SKIP_TEST_QA_CHECK,,}" != "true" ]; then echo "Running qa checks in /docker-entrypoint-initdb.d/qa_checks" - pipenv run python run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks + pushd ../test_data + pipenv shell + popd + python bin/run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks fi From 09cac3c42c4c348b45772f2fbde004ebd7f9da73 Mon Sep 17 00:00:00 2001 From: Thingus Date: Fri, 15 Mar 2024 11:17:33 +0000 Subject: [PATCH 23/31] Ah, the pipfile is in the root --- flowdb/testdata/bin/9900_ingest_synthetic_data.sh | 2 +- flowdb/testdata/bin/9900_ingest_test_data.sh | 2 +- flowdb_testdata.Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh index 8632e1b87c..2915d7d190 100644 --- a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh +++ b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh @@ -79,7 +79,7 @@ else fi if [ "${SKIP_TEST_QA_CHECK}" != "true" ]; then echo "Running QA checks on test data" - pushd ../synthetic_data + pushd .. pipenv shell popd python run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks diff --git a/flowdb/testdata/bin/9900_ingest_test_data.sh b/flowdb/testdata/bin/9900_ingest_test_data.sh index 7c01c22ff9..e1802243ea 100644 --- a/flowdb/testdata/bin/9900_ingest_test_data.sh +++ b/flowdb/testdata/bin/9900_ingest_test_data.sh @@ -36,7 +36,7 @@ fi # &{VAR,,} should lowercase the variable on interpolation if [ "${SKIP_TEST_QA_CHECK,,}" != "true" ]; then echo "Running qa checks in /docker-entrypoint-initdb.d/qa_checks" - pushd ../test_data + pushd .. pipenv shell popd python bin/run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks diff --git a/flowdb_testdata.Dockerfile b/flowdb_testdata.Dockerfile index 7a2f8269dd..1c6fde35f0 100644 --- a/flowdb_testdata.Dockerfile +++ b/flowdb_testdata.Dockerfile @@ -40,7 +40,7 @@ COPY --chown=postgres flowdb/testdata/bin/* /docker-entrypoint-initdb.d/ ADD --chown=postgres flowdb/testdata/test_data/sql/* /docker-entrypoint-initdb.d/sql/testdata/ ADD --chown=postgres flowdb/testdata/test_data/py/* /docker-entrypoint-initdb.d/py/testdata/ ADD --chown=postgres flowdb/testdata/test_data/data/ /docker-entrypoint-initdb.d/data/ -COPY --chown=postgres flowdb/testdata/test_data/data/*.csv /docker-entrypoint-initdb.d/data/csv/ +COPY --chown=postgres flowdb/testdata/test_data/data/*.csv /docker-entrypoint-initdb.d/data/csv/ COPY --chown=postgres flowetl/flowetl/flowetl/qa_checks/qa_checks /docker-entrypoint-initdb.d/qa_checks # Need to make postgres owner of any subdirectories RUN chown -R postgres /docker-entrypoint-initdb.d From c10739af442928f105b502340b8e92683e5390eb Mon Sep 17 00:00:00 2001 From: Thingus Date: Fri, 15 Mar 2024 11:35:54 +0000 Subject: [PATCH 24/31] Lets try an explicity path --- flowdb/testdata/bin/9900_ingest_synthetic_data.sh | 2 +- flowdb/testdata/bin/9900_ingest_test_data.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh index 2915d7d190..b065a5de09 100644 --- a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh +++ b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh @@ -79,7 +79,7 @@ else fi if [ "${SKIP_TEST_QA_CHECK}" != "true" ]; then echo "Running QA checks on test data" - pushd .. + pushd /docker-entrypoint-initdb.d pipenv shell popd python run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks diff --git a/flowdb/testdata/bin/9900_ingest_test_data.sh b/flowdb/testdata/bin/9900_ingest_test_data.sh index e1802243ea..66f0634cb9 100644 --- a/flowdb/testdata/bin/9900_ingest_test_data.sh +++ b/flowdb/testdata/bin/9900_ingest_test_data.sh @@ -36,8 +36,8 @@ fi # &{VAR,,} should lowercase the variable on interpolation if [ "${SKIP_TEST_QA_CHECK,,}" != "true" ]; then echo "Running qa checks in /docker-entrypoint-initdb.d/qa_checks" - pushd .. + pushd /docker-entrypoint-initdb.d pipenv shell popd - python bin/run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks + python run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks fi From 57ca678d4d86fecbd5a839a59726ff3831a508cb Mon Sep 17 00:00:00 2001 From: Thingus Date: Fri, 15 Mar 2024 17:05:57 +0000 Subject: [PATCH 25/31] run_qa_checks now runs on local container --- .../testdata/bin/9900_ingest_synthetic_data.sh | 5 +---- flowdb/testdata/bin/run_qa_checks.py | 18 ++++++++---------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh index b065a5de09..f25c4d2c3e 100644 --- a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh +++ b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh @@ -79,8 +79,5 @@ else fi if [ "${SKIP_TEST_QA_CHECK}" != "true" ]; then echo "Running QA checks on test data" - pushd /docker-entrypoint-initdb.d - pipenv shell - popd - python run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks + pipenv run python run_qa_checks.py qa_checks fi diff --git a/flowdb/testdata/bin/run_qa_checks.py b/flowdb/testdata/bin/run_qa_checks.py index c7addcae9e..89f45618c4 100644 --- a/flowdb/testdata/bin/run_qa_checks.py +++ b/flowdb/testdata/bin/run_qa_checks.py @@ -7,8 +7,8 @@ from itertools import product from pathlib import Path from typing import List -from jinja2 import Environment, PackageLoader, Template -from sqlalchemy import create_engine +from jinja2 import Environment, FileSystemLoader, Template +from sqlalchemy import create_engine, text from sqlalchemy.engine import Engine import os import argparse @@ -63,14 +63,14 @@ def render_qa_check(template: Template, date: date, cdr_type: str) -> str: def get_available_tables(engine: Engine): with engine.begin() as conn: tables = conn.execute( - "SELECT table_name FROM available_tables WHERE has_subscribers" + text("SELECT table_name FROM available_tables WHERE has_subscribers") ) return [t[0] for t in tables.all()] def get_available_dates(engine: Engine): with engine.begin() as conn: - dates = conn.execute("SELECT cdr_date FROM etl.available_dates") + dates = conn.execute(text("SELECT cdr_date FROM etl.available_dates")) return [d[0] for d in dates.all()] @@ -89,13 +89,11 @@ def get_available_dates(engine: Engine): "--event_types", help="Event tables to run qa checks on.", nargs="*" ) args = parser.parse_args() - env = Environment(loader=PathLoader(args.qa_template_path)) + env = Environment(loader=FileSystemLoader(args.template_path)) update_template = env.from_string(update_template_string) db_user = os.environ["POSTGRES_USER"] - db_password = os.environ["POSTGRES_PASSWORD"] - db_port = os.getenv("POSTGRES_PORT", 5432) # For local running - conn_str = f"postgresql://{db_user}:{db_password}@localhost:{db_port}/flowdb" + conn_str = f"postgresql://{db_user}@/flowdb" engine = create_engine(conn_str) dates = get_available_dates(engine) if not args.dates else args.dates @@ -130,7 +128,7 @@ def get_available_dates(engine: Engine): with engine.begin() as conn: for row in qa_rows: - conn.execute(update_template.render(**asdict(row))) + conn.execute(text(update_template.render(**asdict(row)))) - out = conn.execute("SELECT * FROM etl.post_etl_queries") + out = conn.execute(text("SELECT * FROM etl.post_etl_queries LIMIT 10")) print(out.fetchall()) From 0b265e198f6f659057621333bc48b51545c6d3c8 Mon Sep 17 00:00:00 2001 From: Thingus Date: Mon, 18 Mar 2024 11:15:43 +0000 Subject: [PATCH 26/31] More prints + popenv run --- flowdb/testdata/bin/9900_ingest_test_data.sh | 5 +---- flowdb/testdata/bin/run_qa_checks.py | 6 +++++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/flowdb/testdata/bin/9900_ingest_test_data.sh b/flowdb/testdata/bin/9900_ingest_test_data.sh index 66f0634cb9..1beac1cb5e 100644 --- a/flowdb/testdata/bin/9900_ingest_test_data.sh +++ b/flowdb/testdata/bin/9900_ingest_test_data.sh @@ -36,8 +36,5 @@ fi # &{VAR,,} should lowercase the variable on interpolation if [ "${SKIP_TEST_QA_CHECK,,}" != "true" ]; then echo "Running qa checks in /docker-entrypoint-initdb.d/qa_checks" - pushd /docker-entrypoint-initdb.d - pipenv shell - popd - python run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks + pipenv run python run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks fi diff --git a/flowdb/testdata/bin/run_qa_checks.py b/flowdb/testdata/bin/run_qa_checks.py index 89f45618c4..01e9c78395 100644 --- a/flowdb/testdata/bin/run_qa_checks.py +++ b/flowdb/testdata/bin/run_qa_checks.py @@ -90,11 +90,12 @@ def get_available_dates(engine: Engine): ) args = parser.parse_args() env = Environment(loader=FileSystemLoader(args.template_path)) + print(f"Loaded {len(env.list_templates)} templates") update_template = env.from_string(update_template_string) - db_user = os.environ["POSTGRES_USER"] conn_str = f"postgresql://{db_user}@/flowdb" engine = create_engine(conn_str) + print(f"Connecting to flowdb on {conn_str}.") dates = get_available_dates(engine) if not args.dates else args.dates @@ -128,6 +129,9 @@ def get_available_dates(engine: Engine): with engine.begin() as conn: for row in qa_rows: + print( + f"Running {row.type_of_query_or_check} for cdr type {row.cdr_type} date {row.cdr_date}" + ) conn.execute(text(update_template.render(**asdict(row)))) out = conn.execute(text("SELECT * FROM etl.post_etl_queries LIMIT 10")) From 4e837a9cf29215e4755547c90a737f43e713a5a3 Mon Sep 17 00:00:00 2001 From: Thingus Date: Mon, 18 Mar 2024 11:44:35 +0000 Subject: [PATCH 27/31] cd instead of push/popd --- flowdb/testdata/bin/9900_ingest_synthetic_data.sh | 4 ++-- flowdb/testdata/bin/9900_ingest_test_data.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh index f25c4d2c3e..b1a01400cf 100644 --- a/flowdb/testdata/bin/9900_ingest_synthetic_data.sh +++ b/flowdb/testdata/bin/9900_ingest_synthetic_data.sh @@ -23,7 +23,7 @@ fi if [ $count != 0 ]; then echo "Found $count SQL data files in directory." if [ "$(ls -A $DIR)" ]; then - pushd $DIR + cd $DIR for f in *.sql do echo "Running" @@ -31,7 +31,6 @@ if [ $count != 0 ]; then psql --dbname="$POSTGRES_DB" -f $f echo "------------- // Done // ---------------" done - popd else echo "$DIR is empty." fi @@ -78,6 +77,7 @@ else exit 1 fi if [ "${SKIP_TEST_QA_CHECK}" != "true" ]; then + cd /docker-entrypoint-initdb.d echo "Running QA checks on test data" pipenv run python run_qa_checks.py qa_checks fi diff --git a/flowdb/testdata/bin/9900_ingest_test_data.sh b/flowdb/testdata/bin/9900_ingest_test_data.sh index 1beac1cb5e..d9acef8872 100644 --- a/flowdb/testdata/bin/9900_ingest_test_data.sh +++ b/flowdb/testdata/bin/9900_ingest_test_data.sh @@ -19,7 +19,7 @@ fi if [ $count != 0 ]; then echo "Found $count SQL data files in directory." if [ "$(ls -A $DIR)" ]; then - pushd $DIR + cd $DIR for f in *.sql do echo "Running" @@ -27,7 +27,6 @@ if [ $count != 0 ]; then psql --dbname="$POSTGRES_DB" -f $f echo "------------- // Done // ---------------" done - popd else echo "$DIR is empty." fi @@ -36,5 +35,6 @@ fi # &{VAR,,} should lowercase the variable on interpolation if [ "${SKIP_TEST_QA_CHECK,,}" != "true" ]; then echo "Running qa checks in /docker-entrypoint-initdb.d/qa_checks" + cd /docker-entrypoint-initdb.d pipenv run python run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks fi From 8bd422379eeeecafab1486b3e8b80870f31af7c4 Mon Sep 17 00:00:00 2001 From: Thingus Date: Mon, 18 Mar 2024 12:29:37 +0000 Subject: [PATCH 28/31] Minor fixes --- flowdb/testdata/bin/9900_ingest_test_data.sh | 2 +- flowdb/testdata/bin/run_qa_checks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flowdb/testdata/bin/9900_ingest_test_data.sh b/flowdb/testdata/bin/9900_ingest_test_data.sh index d9acef8872..e7e2a4e28b 100644 --- a/flowdb/testdata/bin/9900_ingest_test_data.sh +++ b/flowdb/testdata/bin/9900_ingest_test_data.sh @@ -36,5 +36,5 @@ fi if [ "${SKIP_TEST_QA_CHECK,,}" != "true" ]; then echo "Running qa checks in /docker-entrypoint-initdb.d/qa_checks" cd /docker-entrypoint-initdb.d - pipenv run python run_qa_checks.py /docker-entrypoint-initdb.d/qa_checks + pipenv run python run_qa_checks.py qa_checks fi diff --git a/flowdb/testdata/bin/run_qa_checks.py b/flowdb/testdata/bin/run_qa_checks.py index 01e9c78395..eb05f97de2 100644 --- a/flowdb/testdata/bin/run_qa_checks.py +++ b/flowdb/testdata/bin/run_qa_checks.py @@ -90,7 +90,7 @@ def get_available_dates(engine: Engine): ) args = parser.parse_args() env = Environment(loader=FileSystemLoader(args.template_path)) - print(f"Loaded {len(env.list_templates)} templates") + print(f"Loaded {len(env.list_templates())} templates") update_template = env.from_string(update_template_string) db_user = os.environ["POSTGRES_USER"] conn_str = f"postgresql://{db_user}@/flowdb" From 2b7c409a7ddc5b5da163d3af7d876e18f35c872d Mon Sep 17 00:00:00 2001 From: Jonathan Gray Date: Mon, 18 Mar 2024 14:48:16 +0000 Subject: [PATCH 29/31] Explcit copy of run_qa_checks.py for synth data --- flowdb_synthetic_data.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/flowdb_synthetic_data.Dockerfile b/flowdb_synthetic_data.Dockerfile index 82f67db113..8311594a19 100644 --- a/flowdb_synthetic_data.Dockerfile +++ b/flowdb_synthetic_data.Dockerfile @@ -38,6 +38,7 @@ RUN mkdir -p /docker-entrypoint-initdb.d/sql/syntheticdata/ && \ COPY --chown=postgres flowdb/testdata/bin/9900_ingest_synthetic_data.sh /docker-entrypoint-initdb.d/ COPY --chown=postgres flowdb/testdata/bin/9800_population_density.sql.gz /docker-entrypoint-initdb.d/ +COPY --chown=postgres flowdb/testdata/bin/run_qa_checks.py /docker-entrypoint-initdb.d/ COPY --chown=postgres flowdb/testdata/bin/9910_run_synthetic_dfs_data_generation_script.sh /docker-entrypoint-initdb.d/ COPY --chown=postgres flowdb/testdata/test_data/py/* /docker-entrypoint-initdb.d/py/testdata/ From 9cf261135797dfe6587de9780d2151948df9b015 Mon Sep 17 00:00:00 2001 From: Thingus Date: Mon, 18 Mar 2024 15:37:06 +0000 Subject: [PATCH 30/31] Adding run_qa_checks.py to synth data image --- flowdb_synthetic_data.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/flowdb_synthetic_data.Dockerfile b/flowdb_synthetic_data.Dockerfile index 82f67db113..e7348aa218 100644 --- a/flowdb_synthetic_data.Dockerfile +++ b/flowdb_synthetic_data.Dockerfile @@ -39,6 +39,7 @@ RUN mkdir -p /docker-entrypoint-initdb.d/sql/syntheticdata/ && \ COPY --chown=postgres flowdb/testdata/bin/9900_ingest_synthetic_data.sh /docker-entrypoint-initdb.d/ COPY --chown=postgres flowdb/testdata/bin/9800_population_density.sql.gz /docker-entrypoint-initdb.d/ COPY --chown=postgres flowdb/testdata/bin/9910_run_synthetic_dfs_data_generation_script.sh /docker-entrypoint-initdb.d/ +COPY --chown=postgres flowdb/testdata/bin/run_qa_checks.py /docker-entrypoint-initdb.d/ COPY --chown=postgres flowdb/testdata/test_data/py/* /docker-entrypoint-initdb.d/py/testdata/ COPY --chown=postgres flowdb/testdata/bin/generate_synthetic_data*.py /opt/synthetic_data/ From a346ddd07b73a6ebab29948820aa79ca6348478b Mon Sep 17 00:00:00 2001 From: Thingus Date: Mon, 18 Mar 2024 15:38:19 +0000 Subject: [PATCH 31/31] Removing extra COPY command --- flowdb_synthetic_data.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/flowdb_synthetic_data.Dockerfile b/flowdb_synthetic_data.Dockerfile index c4fb370023..8311594a19 100644 --- a/flowdb_synthetic_data.Dockerfile +++ b/flowdb_synthetic_data.Dockerfile @@ -40,7 +40,6 @@ COPY --chown=postgres flowdb/testdata/bin/9900_ingest_synthetic_data.sh /docker- COPY --chown=postgres flowdb/testdata/bin/9800_population_density.sql.gz /docker-entrypoint-initdb.d/ COPY --chown=postgres flowdb/testdata/bin/run_qa_checks.py /docker-entrypoint-initdb.d/ COPY --chown=postgres flowdb/testdata/bin/9910_run_synthetic_dfs_data_generation_script.sh /docker-entrypoint-initdb.d/ -COPY --chown=postgres flowdb/testdata/bin/run_qa_checks.py /docker-entrypoint-initdb.d/ COPY --chown=postgres flowdb/testdata/test_data/py/* /docker-entrypoint-initdb.d/py/testdata/ COPY --chown=postgres flowdb/testdata/bin/generate_synthetic_data*.py /opt/synthetic_data/