updates 2024-11-15 - flood data errors

enmeshed-analytics · Nov 15, 2024 · 72e69d2 · 72e69d2
1 parent c66931b
commit 72e69d2
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 2 deletions.
diff --git a/...orm_dagster/assets/environment_data_assets/analytics_platform_ea_flood_public_forecast.py b/...orm_dagster/assets/environment_data_assets/analytics_platform_ea_flood_public_forecast.py
@@ -30,7 +30,9 @@ def ea_flood_public_forecast_bronze(context: AssetExecutionContext):
             validation_errors = e.errors()
 
         df = pl.DataFrame(data)
+
         context.log.info(f"Processed {len(df)} records with {len(validation_errors)} validation errors")
+        context.log.info(f"{df.head(5)}")
 
         parquet_buffer = io.BytesIO()
         df.write_parquet(parquet_buffer)
@@ -54,8 +56,12 @@ def ea_flood_public_forecast_silver(context: AssetExecutionContext, ea_flood_pub
     """
     EA Public Forecast flooding data silver bucket
     """
+
+    data = ea_flood_public_forecast_bronze.to_dict()
+    print(data)
+
     # Load data into model
-    flood_risk_data = FloodRiskData.model_validate(ea_flood_public_forecast_bronze.to_dict())
+    flood_risk_data = FloodRiskData.model_validate(data)
 
     # Create flattened dictionary structure
     output = {

diff --git a/analytics_platform_dagster/utils/duckdb_helper/__init__.py b/analytics_platform_dagster/utils/duckdb_helper/__init__.py
diff --git a/analytics_platform_dagster/utils/duckdb_helper/duckdb_helper_open_usrns.py b/analytics_platform_dagster/utils/duckdb_helper/duckdb_helper_open_usrns.py