fix(ingest): continue validation of s3 path_specs even if platform is…

… set (#5951)
datahub-project · Sep 16, 2022 · a14617b · a14617b
1 parent eca73c0
commit a14617b
Show file tree

Hide file tree

Showing 3 changed files with 1,660 additions and 7 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py
@@ -66,10 +66,6 @@ class DataLakeSourceConfig(PlatformSourceConfigBase, EnvBasedSourceConfigBase):
 
     @pydantic.root_validator(pre=False)
     def validate_platform(cls, values: Dict) -> Dict:
-        value = values.get("platform")
-        if value is not None and value != "":
-            return values
-
         if not values.get("path_specs") and not values.get("path_spec"):
             raise ValueError("Either path_specs or path_spec needs to be specified")
 
@@ -96,8 +92,8 @@ def validate_platform(cls, values: Dict) -> Dict:
 
                 platform = "file"
 
-            if values.get("platform", "") != "":
-                if values["platform"] != platform:
+            if values.get("platform", ""):
+                if platform == "s3" and values["platform"] != platform:
                     raise ValueError("all path_spec should belong to the same platform")
             else:
                 values["platform"] = platform