fix(ingest): profiling - Fixing issue with the wrong timestamp stored…

… in check (datahub-project#6978) Co-authored-by: Harshal Sheth <[email protected]>
CybercentreCanada · Feb 1, 2023 · 0baee99 · 0baee99
1 parent df24122
commit 0baee99
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 4 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py
@@ -328,9 +328,9 @@ def get_tables_for_dataset(
                 name=table.table_name,
                 created=table.created,
                 last_altered=datetime.fromtimestamp(
-                    table.last_altered / 1000, tz=timezone.utc
+                    table.get("last_altered") / 1000, tz=timezone.utc
                 )
-                if "last_altered" in table
+                if table.get("last_altered") is not None
                 else table.created,
                 size_in_bytes=table.get("bytes"),
                 rows_count=table.get("row_count"),

diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py
@@ -231,7 +231,7 @@ def generate_wu_from_profile_requests(
             # We don't add to the profiler state if we only do table level profiling as it always happens
             if self.state_handler and not request.profile_table_level_only:
                 self.state_handler.add_to_state(
-                    dataset_urn, int(datetime.utcnow().timestamp() * 1000)
+                    dataset_urn, int(datetime.now().timestamp() * 1000)
                 )
 
             wu = wrap_aspect_as_workunit(

diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py
@@ -107,7 +107,7 @@ def get_workunits(self, databases: List[SnowflakeDatabase]) -> Iterable[WorkUnit
                 # We don't add to the profiler state if we only do table level profiling as it always happens
                 if self.state_handler:
                     self.state_handler.add_to_state(
-                        dataset_urn, int(datetime.utcnow().timestamp() * 1000)
+                        dataset_urn, int(datetime.now().timestamp() * 1000)
                     )
 
                 yield self.wrap_aspect_as_workunit(