fix: For FDA Food Enforcement, Resolve invalid source DateTime data.

nlarge-google · Oct 13, 2022 · 2fe2249 · 2fe2249
1 parent 802cff6
commit 2fe2249
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 4 deletions.
diff --git a/datasets/fda_food/pipelines/_images/run_csv_transform_kub/csv_transform.py b/datasets/fda_food/pipelines/_images/run_csv_transform_kub/csv_transform.py
@@ -206,7 +206,6 @@ def resolve_date_format(
         df[col] = df[col].apply(
             lambda x: convert_dt_format(str(x), from_format, to_format, is_date)
         )
-
     return df
 
 
@@ -216,6 +215,8 @@ def convert_dt_format(
     rtnval = "<initial_value>"
     if not dt_str or str(dt_str).lower() == "nan" or str(dt_str).lower() == "nat":
         rtnval = ""
+    elif len(dt_str) > 20:
+        rtnval = ""
     elif len(dt_str.strip()) == 10:
         # if there is no time format
         rtnval = dt_str + " 00:00:00"
@@ -239,7 +240,6 @@ def convert_dt_format(
         from_format = "%Y-%m-%d " + from_format.strip().split(" ")[1]
     else:
         dt_str = "<blank>"
-
     return rtnval
 
 

diff --git a/datasets/fda_food/pipelines/food_enforcement/food_enforcement_dag.py b/datasets/fda_food/pipelines/food_enforcement/food_enforcement_dag.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Google LLC
+# Copyright 2022 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/datasets/fda_food/pipelines/food_events/food_events_dag.py b/datasets/fda_food/pipelines/food_events/food_events_dag.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Google LLC
+# Copyright 2022 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.