From b98cd0c894635fc4cb7c8fcf26f3dbb00ce2f11e Mon Sep 17 00:00:00 2001 From: Thingus Date: Fri, 17 Feb 2023 16:40:39 +0000 Subject: [PATCH 1/3] Look for COUNT instead of just first row in data_present_sensor --- flowetl/flowetl/flowetl/sensors/data_present_sensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowetl/flowetl/flowetl/sensors/data_present_sensor.py b/flowetl/flowetl/flowetl/sensors/data_present_sensor.py index b808c9fd12..d93315253d 100644 --- a/flowetl/flowetl/flowetl/sensors/data_present_sensor.py +++ b/flowetl/flowetl/flowetl/sensors/data_present_sensor.py @@ -6,6 +6,6 @@ DataPresentSensor = fixed_sql_operator( class_name="DataPresentSensor", - sql="SELECT * FROM {{ staging_table }} LIMIT 1;", + sql="SELECT COUNT (*) FROM {{ staging_table }} LIMIT 100;", is_sensor=True, ) From 7962add71c187d6c2eebf5c5b6ddab047a486e1e Mon Sep 17 00:00:00 2001 From: Thingus Date: Mon, 20 Feb 2023 09:22:19 +0000 Subject: [PATCH 2/3] Using SELECT EXISTS instead of COUNT --- flowetl/flowetl/flowetl/sensors/data_present_sensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowetl/flowetl/flowetl/sensors/data_present_sensor.py b/flowetl/flowetl/flowetl/sensors/data_present_sensor.py index d93315253d..467dd9b7c5 100644 --- a/flowetl/flowetl/flowetl/sensors/data_present_sensor.py +++ b/flowetl/flowetl/flowetl/sensors/data_present_sensor.py @@ -6,6 +6,6 @@ DataPresentSensor = fixed_sql_operator( class_name="DataPresentSensor", - sql="SELECT COUNT (*) FROM {{ staging_table }} LIMIT 100;", + sql="SELECT EXISTS(SELECT * FROM {{ staging_table }} LIMIT 1);", is_sensor=True, ) From 9f72d530598b942d4da23dc6658ef2c17ce59129 Mon Sep 17 00:00:00 2001 From: Thingus Date: Mon, 20 Feb 2023 09:48:31 +0000 Subject: [PATCH 3/3] Changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab1be35911..372106f6a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Fixed + +- Null values in first column of first row of ingested data no longer cause flowetl to skip ingestion [#5090](https://github.com/Flowminder/FlowKit/issues/5090) + ## [1.18.2] ## Fixed