Merge branch 'develop' into winter2024research

.github/pull_request_template.md

@@ -6,8 +6,8 @@ Pull request closes #_
 _List the steps to test the PR_
 These steps are generic, please adjust as necessary.
 ```
-cd tdrs-frontend && docker-compose -f docker-compose.yml -f docker-compose.local.yml up -d
-cd tdrs-backend && docker-compose -f docker-compose.yml -f docker-compose.local.yml up -d 
+cd tdrs-frontend && docker-compose up --build
+cd tdrs-backend && docker-compose up --build
 ```
 
 1. Open http://localhost:3000/ and sign in.

tdrs-backend/tdpservice/data_files/views.py

@@ -146,7 +146,7 @@ def download(self, request, pk=None):
     def download_error_report(self, request, pk=None):
         """Generate and return the parsing error report xlsx."""
         datafile = self.get_object()
-        all_errors = ParserError.objects.filter(file=datafile)
+        all_errors = ParserError.objects.filter(file=datafile, deprecated=False)
         is_s3_s4 = (DataFile.Section.AGGREGATE_DATA in datafile.section or
                     DataFile.Section.STRATUM_DATA in datafile.section)
         filtered_errors = get_prioritized_queryset(all_errors, is_s3_s4)

tdrs-backend/tdpservice/parsers/aggregates.py

@@ -21,6 +21,7 @@ def case_aggregates_by_month(df, dfs_status):
     schema_models = [model for model in schema_models_dict.values()]
 
     aggregate_data = {"months": [], "rejected": 0}
+    all_errors = ParserError.objects.filter(file=df, deprecated=False)
     for month in month_list:
         total = 0
         cases_with_errors = 0
@@ -46,8 +47,7 @@ def case_aggregates_by_month(df, dfs_status):
             case_numbers = case_numbers.union(curr_case_numbers)
 
         total += len(case_numbers)
-        cases_with_errors += ParserError.objects.filter(file=df, case_number__in=case_numbers)\
-            .distinct('case_number').count()
+        cases_with_errors += all_errors.filter(case_number__in=case_numbers).distinct('case_number').count()
         accepted = total - cases_with_errors
 
         aggregate_data['months'].append({"month": month,
@@ -57,8 +57,8 @@ def case_aggregates_by_month(df, dfs_status):
     error_type_query = Query(error_type=ParserErrorCategoryChoices.PRE_CHECK) | \
         Query(error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY)
 
-    aggregate_data['rejected'] = ParserError.objects.filter(error_type_query, file=df)\
-        .distinct("row_number").exclude(row_number=0).count()
+    aggregate_data['rejected'] = all_errors.filter(error_type_query).distinct("row_number")\
+        .exclude(row_number=0).count()
 
     return aggregate_data
 
@@ -70,7 +70,7 @@ def total_errors_by_month(df, dfs_status):
 
     total_errors_data = {"months": []}
 
-    errors = ParserError.objects.all().filter(file=df)
+    errors = ParserError.objects.all().filter(file=df, deprecated=False)
 
     for month in month_list:
         if dfs_status == "Rejected":

tdrs-backend/tdpservice/parsers/case_consistency_validator.py

@@ -9,6 +9,7 @@
 from tdpservice.parsers.validators.util import ValidationErrorArgs
 from tdpservice.parsers.validators.category3 import format_error_context
 import logging
+import warnings
 
 logger = logging.getLogger(__name__)
 
@@ -54,7 +55,7 @@ def __get_error_context(self, field_name, schema):
                                          )
         return format_error_context(error_args)
 
-    def __generate_and_add_error(self, schema, record, field, line_num, msg):
+    def __generate_and_add_error(self, schema, record, field, line_num, msg, deprecated=False):
         """Generate a ParserError and add it to the `generated_errors` list."""
         err = self.generate_error(
             error_category=ParserErrorCategoryChoices.CASE_CONSISTENCY,
@@ -63,6 +64,7 @@ def __generate_and_add_error(self, schema, record, field, line_num, msg):
             record=record,
             field=field,
             error_message=msg,
+            deprecated=deprecated
         )
         self.generated_errors.append(err)
 
@@ -340,11 +342,14 @@ def __validate_s1_records_are_related(self):
 
     def __validate_case_closure_employment(self, t4, t5s, error_msg):
         """
-        Validate case closure.
+        Validate case closure. (DEPRECATED, always returns zero.).
 
         If case closure reason = 01:employment, then at least one person on
         the case must have employment status = 1:Yes in the same month.
         """
+        warnings.warn(("No longer considered a category four failure. "
+                      "Records are serialized even if this error is generated."), DeprecationWarning,
+                      stacklevel=2)
         num_errors = 0
         t4_record, t4_schema, line_num = t4
 
@@ -362,19 +367,23 @@ def __validate_case_closure_employment(self, t4, t5s, error_msg):
                 t4_record,
                 "EMPLOYMENT_STATUS",
                 line_num,
-                error_msg
+                error_msg,
+                deprecated=True
             )
             num_errors += 1
 
-        return num_errors
+        return 0
 
     def __validate_case_closure_ftl(self, t4, t5s, error_msg):
         """
-        Validate case closure.
+        Validate case closure. (DEPRECATED, always returns zero.).
 
         If closure reason = FTL, then at least one person who is HoH
         or spouse of HoH on case must have FTL months >=60.
         """
+        warnings.warn(("No longer considered a category four failure. "
+                      "Records are serialized even if this error is generated."), DeprecationWarning,
+                      stacklevel=2)
         num_errors = 0
         t4_record, t4_schema, line_num = t4
 
@@ -393,11 +402,12 @@ def __validate_case_closure_ftl(self, t4, t5s, error_msg):
                 t4_record,
                 "COUNTABLE_MONTH_FED_TIME",
                 line_num,
-                error_msg
+                error_msg,
+                deprecated=True
             )
             num_errors += 1
 
-        return num_errors
+        return 0
 
     def __validate_s2_records_are_related(self):
         """
@@ -466,6 +476,10 @@ def __validate_s2_records_are_related(self):
         return num_errors
 
     def __validate_t5_atd_and_ssi(self):
+        """Validate aid totally disabled and SSI. (DEPRECATED, always returns zero.)."""
+        warnings.warn(("No longer considered a category four failure. "
+                      "Records are serialized even if this error is generated."), DeprecationWarning,
+                      stacklevel=2)
         num_errors = 0
         t4s, t4_model_name, t5s, t5_model_name = self.__get_s2_triplets_and_names()
 
@@ -492,7 +506,8 @@ def __validate_t5_atd_and_ssi(self):
                     msg=(
                         f"{t5_model_name} Adults in territories must have a valid "
                         f"value for {self.__get_error_context('REC_AID_TOTALLY_DISABLED', schema)}."
-                    )
+                    ),
+                    deprecated=True
                 )
                 num_errors += 1
             elif is_state and rec_atd == 1:
@@ -504,7 +519,8 @@ def __validate_t5_atd_and_ssi(self):
                     msg=(
                         f"{t5_model_name} People in states should not have a value "
                         f"of 1 for {self.__get_error_context('REC_AID_TOTALLY_DISABLED', schema)}."
-                    )
+                    ),
+                    deprecated=True
                 )
                 num_errors += 1
 
@@ -517,7 +533,8 @@ def __validate_t5_atd_and_ssi(self):
                     msg=(
                         f"{t5_model_name} People in territories must have value = 2:No for "
                         f"{self.__get_error_context('REC_SSI', schema)}."
-                    )
+                    ),
+                    deprecated=True
                 )
                 num_errors += 1
             elif is_state and family_affiliation == 1 and rec_ssi not in {1, 2}:
@@ -529,8 +546,9 @@ def __validate_t5_atd_and_ssi(self):
                     msg=(
                         f"{t5_model_name} People in states must have a valid value for "
                         f"{self.__get_error_context('REC_SSI', schema)}."
-                    )
+                    ),
+                    deprecated=True
                 )
                 num_errors += 1
 
-        return num_errors
+        return 0

tdrs-backend/tdpservice/parsers/migrations/0012_parsererror_deprecated.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.15 on 2024-12-18 16:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('parsers', '0011_parsererror_row_number_optional'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='parsererror',
+            name='deprecated',
+            field=models.BooleanField(default=False),
+        ),
+    ]

tdrs-backend/tdpservice/parsers/models.py

@@ -44,6 +44,8 @@ class Meta:
     created_at = models.DateTimeField(auto_now_add=True)
     fields_json = models.JSONField(null=True)
 
+    deprecated = models.BooleanField(default=False)
+
     @property
     def rpt_month_name(self):
         """Return the month name."""
@@ -110,7 +112,7 @@ def get_status(self):
         if self.status != DataFileSummary.Status.PENDING:
             return self.status
 
-        errors = ParserError.objects.filter(file=self.datafile)
+        errors = ParserError.objects.filter(file=self.datafile, deprecated=False)
 
         # excluding row-level pre-checks and trailer pre-checks.
         precheck_errors = errors.filter(error_type=ParserErrorCategoryChoices.PRE_CHECK)\

tdrs-backend/tdpservice/parsers/parse.py

@@ -61,43 +61,43 @@ def parse_datafile(datafile, dfs):
 
     # Validate tribe code in submission across program type and fips code
     generate_error = util.make_generate_parser_error(datafile, 1)
-    tribe_is_valid, tribe_error = category1.validate_tribe_fips_program_agree(
+    tribe_result = category1.validate_tribe_fips_program_agree(
         header['program_type'],
         field_values["tribe_code"],
         field_values["state_fips"],
         generate_error
     )
 
-    if not tribe_is_valid:
+    if not tribe_result.valid:
         logger.info(f"Tribe Code ({field_values['tribe_code']}) inconsistency with Program Type " +
                     f"({header['program_type']}) and FIPS Code ({field_values['state_fips']}).",)
-        errors['header'] = [tribe_error]
-        bulk_create_errors({1: [tribe_error]}, 1, flush=True)
+        errors['header'] = [tribe_result.error]
+        bulk_create_errors({1: [tribe_result.error]}, 1, flush=True)
         return errors
 
     # Ensure file section matches upload section
-    section_is_valid, section_error = category1.validate_header_section_matches_submission(
+    section_result = category1.validate_header_section_matches_submission(
         datafile,
         get_section_reference(program_type, section),
         util.make_generate_parser_error(datafile, 1)
     )
 
-    if not section_is_valid:
-        logger.info(f"Preparser Error -> Section is not valid: {section_error.error_message}")
-        errors['document'] = [section_error]
-        unsaved_parser_errors = {1: [section_error]}
+    if not section_result.valid:
+        logger.info(f"Preparser Error -> Section is not valid: {section_result.error}")
+        errors['document'] = [section_result.error]
+        unsaved_parser_errors = {1: [section_result.error]}
         bulk_create_errors(unsaved_parser_errors, 1, flush=True)
         return errors
 
-    rpt_month_year_is_valid, rpt_month_year_error = category1.validate_header_rpt_month_year(
+    rpt_month_year_result = category1.validate_header_rpt_month_year(
         datafile,
         header,
         util.make_generate_parser_error(datafile, 1)
     )
-    if not rpt_month_year_is_valid:
-        logger.info(f"Preparser Error -> Rpt Month Year is not valid: {rpt_month_year_error.error_message}")
-        errors['document'] = [rpt_month_year_error]
-        unsaved_parser_errors = {1: [rpt_month_year_error]}
+    if not rpt_month_year_result.valid:
+        logger.info(f"Preparser Error -> Rpt Month Year is not valid: {rpt_month_year_result.error}")
+        errors['document'] = [rpt_month_year_result.error]
+        unsaved_parser_errors = {1: [rpt_month_year_result.error]}
         bulk_create_errors(unsaved_parser_errors, 1, flush=True)
         return errors
 

tdrs-backend/tdpservice/parsers/row_schema.py

@@ -98,22 +98,23 @@ def run_preparsing_validators(self, line, generate_error):
                 friendly_name=field.friendly_name if field else 'record type',
                 item_num=field.item if field else '0',
             )
-            validator_is_valid, validator_error = validator(line, eargs)
-            is_valid = False if not validator_is_valid else is_valid
+            result = validator(line, eargs)
+            is_valid = False if not result.valid else is_valid
 
             is_quiet_preparser_errors = (
                 self.quiet_preparser_errors
                 if type(self.quiet_preparser_errors) is bool
                 else self.quiet_preparser_errors(line)
             )
-            if validator_error and not is_quiet_preparser_errors:
+            if result.error and not is_quiet_preparser_errors:
                 errors.append(
                     generate_error(
                         schema=self,
                         error_category=ParserErrorCategoryChoices.PRE_CHECK,
-                        error_message=validator_error,
+                        error_message=result.error,
                         record=None,
-                        field="Record_Type"
+                        field="Record_Type",
+                        deprecated=result.deprecated,
                     )
                 )
 
@@ -152,16 +153,17 @@ def run_field_validators(self, instance, generate_error):
             should_validate = not field.required and not is_empty
             if (field.required and not is_empty) or should_validate:
                 for validator in field.validators:
-                    validator_is_valid, validator_error = validator(value, eargs)
-                    is_valid = False if (not validator_is_valid and not field.ignore_errors) else is_valid
-                    if validator_error:
+                    result = validator(value, eargs)
+                    is_valid = False if (not result.valid and not field.ignore_errors) else is_valid
+                    if result.error:
                         errors.append(
                             generate_error(
                                 schema=self,
                                 error_category=ParserErrorCategoryChoices.FIELD_VALUE,
-                                error_message=validator_error,
+                                error_message=result.error,
                                 record=instance,
-                                field=field
+                                field=field,
+                                deprecated=result.deprecated
                             )
                         )
             elif field.required:
@@ -187,18 +189,19 @@ def run_postparsing_validators(self, instance, generate_error):
         errors = []
 
         for validator in self.postparsing_validators:
-            validator_is_valid, validator_error, field_names = validator(instance, self)
-            is_valid = False if not validator_is_valid else is_valid
-            if validator_error:
+            result = validator(instance, self)
+            is_valid = False if not result.valid else is_valid
+            if result.error:
                 # get field from field name
-                fields = [self.get_field_by_name(name) for name in field_names]
+                fields = [self.get_field_by_name(name) for name in result.field_names]
                 errors.append(
                     generate_error(
                         schema=self,
                         error_category=ParserErrorCategoryChoices.VALUE_CONSISTENCY,
-                        error_message=validator_error,
+                        error_message=result.error,
                         record=instance,
                         field=fields,
+                        deprecated=result.deprecated
                     )
                 )