From 40a0ebac47a01174081b44e39181f6046ccf8fbb Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Tue, 7 Feb 2023 03:10:36 +0530 Subject: [PATCH] feat(ingest/s3): use latest file to infer schema metadata (#7202) Co-authored-by: John Joyce --- metadata-ingestion/src/datahub/ingestion/source/s3/source.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py index 142c449f404d02..781fcfbc26cc7f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py @@ -752,6 +752,9 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: table_dict[table_data.table_path].timestamp < table_data.timestamp ): + table_dict[ + table_data.table_path + ].full_path = table_data.full_path table_dict[ table_data.table_path ].timestamp = table_data.timestamp