Merge pull request #6448 from Flowminder/fix-no-poke

Fix no poke

CHANGELOG.md

@@ -7,8 +7,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [Unreleased]
 
 ### Added
+- FlowETL sensor `NRowsPresentSensor` which checks for a specified minimum number of rows.
 
 ### Changed
+- `ForeignStagingTableOperator` will now error if the underlying file cannot be read or the command returns an error. [#5763](https://github.com/Flowminder/FlowKit/issues/5763)
 
 ### Fixed
 

flowetl/flowetl/flowetl/operators/create_foreign_staging_table_operator.py

@@ -66,6 +66,7 @@ def __init__(
                    escape '{{{{ params.escape }}}}'
                    {{% if params.encoding is defined %}}, {{{{ params.encoding }}}} {{% endif %}} 
                    );
+            SELECT EXISTS(SELECT * FROM {{{{ staging_table }}}} LIMIT 1); 
                 """
         fields_string = ",\n\t".join(
             f"{field_name} {field_type.upper()}"

flowetl/flowetl/flowetl/sensors/n_rows_present_sensor.py

@@ -0,0 +1,12 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from flowetl.mixins.fixed_sql_with_params_mixin import fixed_sql_operator_with_params
+
+NRowsPresentSensor = fixed_sql_operator_with_params(
+    class_name="NRowsPresentSensor",
+    sql="SELECT EXISTS(SELECT * FROM {{ staging_table }} LIMIT 1 (OFFSET {{ params.minimum_rows }} - 1));",
+    is_sensor=True,
+    params=["minimum_rows"],
+)