Skip to content

Commit

Permalink
fix(ingest): catch errors when profiling for sample values (#6194)
Browse files Browse the repository at this point in the history
  • Loading branch information
mayurinehate authored Oct 15, 2022
1 parent 43f7b56 commit 3fc6364
Showing 1 changed file with 13 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -460,16 +460,28 @@ def _get_dataset_column_histogram(
def _get_dataset_column_sample_values(
self, column_profile: DatasetFieldProfileClass, column: str
) -> None:
if self.config.include_field_sample_values:
if not self.config.include_field_sample_values:
return

try:
# TODO do this without GE
self.dataset.set_config_value("interactive_evaluation", True)

res = self.dataset.expect_column_values_to_be_in_set(
column, [], result_format="SUMMARY"
).result

column_profile.sampleValues = [
str(v) for v in res["partial_unexpected_list"]
]
except Exception as e:
logger.debug(
f"Caught exception while attempting to get sample values for column {column}. {e}"
)
self.report.report_warning(
"Profiling - Unable to get column sample values",
f"{self.dataset_name}.{column}",
)

def generate_dataset_profile( # noqa: C901 (complexity)
self,
Expand Down

0 comments on commit 3fc6364

Please sign in to comment.