feat(ingest/s3): use latest file to infer schema metadata (datahub-pr…

…oject#7202) Co-authored-by: John Joyce <[email protected]>
ericyomi · Feb 8, 2023 · 40a0eba · 40a0eba
1 parent e3a64f5
commit 40a0eba
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py
@@ -752,6 +752,9 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]:
                             table_dict[table_data.table_path].timestamp
                             < table_data.timestamp
                         ):
+                            table_dict[
+                                table_data.table_path
+                            ].full_path = table_data.full_path
                             table_dict[
                                 table_data.table_path
                             ].timestamp = table_data.timestamp