Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
keep track of record count
Browse files Browse the repository at this point in the history
lchen-2101 committed Jan 28, 2025
1 parent 3aadc82 commit d6bd140
Showing 2 changed files with 3 additions and 1 deletion.
1 change: 1 addition & 0 deletions src/regtech_data_validator/validation_results.py
Original file line number Diff line number Diff line change
@@ -26,3 +26,4 @@ class ValidationResults(object):
is_valid: bool
findings: pl.DataFrame
phase: ValidationPhase
record_count: int = 0
3 changes: 2 additions & 1 deletion src/regtech_data_validator/validator.py
Original file line number Diff line number Diff line change
@@ -310,7 +310,7 @@ def validate_register_level(context: Dict[str, str] | None, all_uids: List[str])
return results


def validate_chunk(schema, df, total_count, row_start, max_errors, process_errors, checks):
def validate_chunk(schema, df: pl.DataFrame, total_count, row_start, max_errors, process_errors, checks):
# print(f"Start UID: {df['uid'][0]}, Last UID: {df['uid'][-1]}", flush=True)
validation_results = validate(schema, df, row_start, process_errors)
if process_errors and not validation_results.is_empty():
@@ -320,6 +320,7 @@ def validate_chunk(schema, df, total_count, row_start, max_errors, process_error

error_counts, warning_counts = get_scope_counts(validation_results)
results = ValidationResults(
record_count=df.height,
error_counts=error_counts,
warning_counts=warning_counts,
is_valid=((error_counts.total_count + warning_counts.total_count) == 0),

0 comments on commit d6bd140

Please sign in to comment.