Merge pull request #4647 from GSA-TTS/main

.github/workflows/deploy-application.yml

@@ -102,14 +102,6 @@ jobs:
           cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml
           command: bin/ops/deploy.sh
 
-      - name: Load historical data
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          command: cf run-task gsa-fac -k 6G -m 1G --name load_data --command "./load_data.sh"
 
       # This has to happen after an application deployment because the manifest (currently) is responsible
       # for binding the "logdrain service" to the "gsa-fac application". This also needs to be done

.github/workflows/deploy-development.yml

@@ -78,11 +78,3 @@ jobs:
     with:
       url: "https://fac-dev.app.cloud.gov/"
 
-  generate-e2e-test-data:
-    needs:
-      - deploy-dev
-    name:
-    uses: ./.github/workflows/end-to-end-test-data-generator.yml
-    secrets: inherit
-    with:
-      environment: "dev"

.github/workflows/testing-from-build.yml

@@ -33,10 +33,6 @@ jobs:
         working-directory: ./backend
         run: docker compose -f docker-compose.yml up -d
 
-      - name: Load historical data
-        working-directory: ./backend
-        run: docker compose run web ./load_data.sh
-
       - name: Run Django test suite
         working-directory: ./backend
         run: docker compose -f docker-compose.yml run web bash -c 'coverage run --parallel-mode --concurrency=multiprocessing manage.py test --parallel && coverage combine && coverage report -m --fail-under=85 && coverage xml -o coverage.xml'

.github/workflows/testing-from-ghcr.yml

@@ -34,10 +34,6 @@ jobs:
         working-directory: ./backend
         run: docker compose -f docker-compose-web.yml up -d
 
-      - name: Load historical data
-        working-directory: ./backend
-        run: docker compose -f docker-compose-web.yml run web ./load_data.sh
-
       - name: Run Django test suite
         working-directory: ./backend
         run: |

backend/Makefile

@@ -64,9 +64,6 @@ docker-migrate:
 		python manage.py makemigrations &&\
 		python manage.py migrate'
 
-docker-load-data:
-	docker compose up -d && docker compose run web ./load_data.sh
-
 # Run Django tests with docker
 docker-test:
 	docker compose build

backend/audit/context.py

@@ -0,0 +1,18 @@
+import contextvars
+from contextlib import contextmanager
+
+
+current_sac = contextvars.ContextVar("current_sac", default=None)
+
+
+@contextmanager
+def set_sac_to_context(sac):
+    reference = current_sac.set(sac)
+    try:
+        yield
+    finally:
+        current_sac.reset(reference)
+
+
+def get_sac_from_context():
+    return current_sac.get(None)

backend/audit/intakelib/checks/check_finding_reference_year.py

@@ -0,0 +1,62 @@
+import logging
+from django.core.exceptions import ValidationError
+from audit.intakelib.intermediate_representation import (
+    get_range_by_name,
+)
+from audit.intakelib.common import (
+    get_message,
+    build_cell_error_tuple,
+)
+
+from audit.context import get_sac_from_context
+from audit.intakelib.common.util import get_range_start_row
+
+logger = logging.getLogger(__name__)
+
+
+# DESCRIPTION
+# Finding references should be in 20##-### format where the first four
+# digits match the audit_year.
+
+
+def finding_reference_year(ir, is_gsa_migration=False):
+    references = get_range_by_name(ir, "reference_number")
+    range_start = int(get_range_start_row(references))
+    errors = []
+    sac = get_sac_from_context()
+    if is_gsa_migration or sac and sac.general_information is None:
+        # In real use cases, no report can be created if general_information is missing, as it is a required field.
+        # The condition sac.general_information is None can occur only in test cases
+        # where general_information has been ignored purposefully (like in test_workbooks_should_pass.py).
+        return
+    elif sac is None:
+        raise ValidationError(
+            (
+                "",
+                "",
+                "Workbook Validation Failed",
+                {
+                    "text": "The workbook cannot be validated at the moment. Please contact the helpdesk for assistance.",
+                    "link": "Intake checks: no link defined",
+                },
+            )
+        )
+    audit_date = sac.general_information["auditee_fiscal_period_end"]
+    audit_year = int(audit_date.split("-")[0])
+    for index, reference in enumerate(references["values"]):
+        year = int(reference.split("-")[0])
+        if audit_year != year:
+            errors.append(
+                build_cell_error_tuple(
+                    ir,
+                    references,
+                    index,
+                    get_message("check_invalid_finding_reference_year").format(
+                        reference, index + range_start, audit_year
+                    ),
+                )
+            )
+
+    if len(errors) > 0:
+        logger.info("Raising a validation error.")
+        raise ValidationError(errors)

backend/audit/intakelib/checks/runners.py

@@ -56,6 +56,7 @@
 from .check_findings_grid_validation import findings_grid_validation
 from .check_finding_prior_references_pattern import prior_references_pattern
 from .check_finding_reference_pattern import finding_reference_pattern
+from .check_finding_reference_year import finding_reference_year
 from .check_aln_prefix_pattern import aln_agency_prefix
 
 logger = logging.getLogger(__name__)
@@ -111,6 +112,7 @@
     award_references_pattern,
     prior_references_pattern,
     finding_reference_pattern,
+    finding_reference_year,
     no_repeat_findings,
     findings_grid_validation,
     # See ticket #4385 for more information on why this check is disabled

backend/audit/intakelib/common/error_messages.py

@@ -69,6 +69,7 @@
     "check_aln_three_digit_extension_invalid": "The three digit extension should follow one of these formats: ###, RD#, or U##, where # represents a number",
     "check_prior_references_invalid": "Prior references must be <b>N/A</b> or a comma-separated list of values in the format <b>20##-###</b>, for example, <b>2019-001, 2019-002</b>",
     "check_finding_reference_invalid": "Finding references must be in the format <b>20##-###</b> where the first four digits are a year after 2010, for example, <b>2019-001, 2019-002</b>",
+    "check_invalid_finding_reference_year": "The reference year in the finding reference <b>{}</b> declared in row {} does not match the audit year <b>{}</b>",
     "check_award_references_invalid": "Combining award references of 4 and 5-digit lengths is not allowed. If needed, zero-pad this number to make it 5 digits",
     "check_aln_prefix_invalid": "The federal agency prefix should be a two-digit value, for example, <b>10</b>",
     "check_additional_award_identification_present": "Missing additional award identification",

backend/audit/test_check_finding_reference_year.py

@@ -0,0 +1,90 @@
+from unittest.mock import Mock
+from django.test import SimpleTestCase
+from django.core.exceptions import ValidationError
+from audit.intakelib.checks.check_finding_reference_year import finding_reference_year
+from audit.context import set_sac_to_context
+
+
+class TestFindingReferenceYear(SimpleTestCase):
+    def setUp(self):
+        self.ir = [
+            {
+                "name": "Form",
+                "ranges": [
+                    {
+                        "name": "reference_number",
+                        "start_cell": {"column": "A", "row": "2"},
+                        "end_cell": {"column": "A", "row": "20001"},
+                        "values": ["2022-001", "2022-002", "2022-003"],
+                    },
+                ],
+            }
+        ]
+        self.mock_sac = Mock()
+
+    def test_success(self):
+        """
+        Test case where all finding reference years match the audit year.
+        """
+        self.mock_sac.general_information = {
+            "auditee_fiscal_period_end": "2022-12-31",
+            "auditee_uei": "UEI",
+        }
+        with set_sac_to_context(self.mock_sac):
+
+            errors = finding_reference_year(self.ir)
+
+            self.assertEqual(errors, None)
+
+    def test_mismatched_years(self):
+        """
+        Test case where finding reference years do not match the audit year.
+        """
+        self.mock_sac.general_information = {
+            "auditee_fiscal_period_end": "2023-12-31",
+            "auditee_uei": "UEI",
+        }
+        with set_sac_to_context(self.mock_sac):
+            with self.assertRaises(ValidationError) as context:
+                finding_reference_year(self.ir)
+
+        errors = context.exception.args[0]
+        self.assertEqual(len(errors), 3)  # Three mismatches
+        self.assertIn("2022-001", errors[0][3]["text"])
+        self.assertIn("2022-002", errors[1][3]["text"])
+        self.assertIn("2022-003", errors[2][3]["text"])
+
+    def test_gsa_migration(self):
+        """
+        Test case where is_gsa_migration is True and no validation is performed.
+        """
+        errors = finding_reference_year(self.ir, is_gsa_migration=True)
+
+        self.assertEqual(errors, None)
+
+    def test_auditee_uei_missing(self):
+        """
+        Test case where auditee_uei is None and the function returns without validation.
+        """
+        self.mock_sac.general_information = {
+            "auditee_fiscal_period_end": "2022-12-31",
+            "auditee_uei": None,
+        }
+        with set_sac_to_context(self.mock_sac):
+            errors = finding_reference_year(self.ir)
+            self.assertEqual(errors, None)
+
+    def test_sac_is_none(self):
+        """
+        Test case where sac is None and a ValidationError is raised.
+        """
+        with set_sac_to_context(None):
+            with self.assertRaises(ValidationError) as context:
+                finding_reference_year(self.ir)
+
+            error = context.exception.args[0]
+            self.assertEqual(error[2], "Workbook Validation Failed")
+            self.assertEqual(
+                error[3]["text"],
+                "The workbook cannot be validated at the moment. Please contact the helpdesk for assistance.",
+            )

backend/audit/test_workbooks_should_pass.py

@@ -1,9 +1,10 @@
-from django.test import SimpleTestCase
+from django.test import TestCase
 import os
 from functools import reduce
 import re
-
-
+from audit.context import set_sac_to_context
+from model_bakery import baker
+from .models import SingleAuditChecklist
 from audit.intakelib import (
     extract_additional_eins,
     extract_additional_ueis,
@@ -76,12 +77,14 @@ def process_workbook_set(workbook_set_path, is_gsa_migration=True):
                     raise Exception(msg)
 
 
-class PassingWorkbooks(SimpleTestCase):
+class PassingWorkbooks(TestCase):
     def test_passing_workbooks(self):
         workbook_sets = reduce(
             os.path.join, ["audit", "fixtures", "workbooks", "should_pass"]
         )
-        for dirpath, dirnames, _ in os.walk(workbook_sets):
-            for workbook_set in dirnames:
-                print("Walking ", workbook_set)
-                process_workbook_set(os.path.join(dirpath, workbook_set))
+        sac = baker.make(SingleAuditChecklist)
+        with set_sac_to_context(sac):
+            for dirpath, dirnames, _ in os.walk(workbook_sets):
+                for workbook_set in dirnames:
+                    print("Walking ", workbook_set)
+                    process_workbook_set(os.path.join(dirpath, workbook_set))

backend/audit/verify_status.py

@@ -17,8 +17,8 @@ def verify_status(status):
     Decorator to be applied to view request methods (i.e. get, post) to verify
     that the submission is in the correct state before allowing the user to
     proceed. An incorrect status usually happens via direct URL access. If the
-    given status does not match the submission's, it will redirect them back to
-    the submission progress page.
+    given status(es) do(es) not match the submission's, it will redirect them back to
+    the submission progress page. Accepts either a str or a [str].
     """
 
     def decorator_verify_status(request_method):
@@ -30,8 +30,10 @@ def verify(view, request, *args, **kwargs):
             except SingleAuditChecklist.DoesNotExist:
                 raise PermissionDenied("You do not have access to this audit.")
 
-            # Return to checklist, the Audit is not in the correct state.
-            if sac.submission_status != status:
+            statuses = status if isinstance(status, list) else [status]
+
+            if sac.submission_status not in statuses:
+                # Return to checklist, the audit is not in the correct state
                 logger.warning(
                     f"Expected submission status {status} but it's currently {sac.submission_status}"
                 )