New management command - Manually move "submitted" report to dissemin…

…ation (#4351)

* Create move_to_disseminated.py

New management command file for attempting to move a SAC to dissemination. This is the first commit on this feature.

Command is: `python manage.py move_to_disseminated --report_id ID_GOES_HERE`. This will look for a SAC with the `report_id` that you entered, and attempt to move it to the `disseminated` status - ONLY if it is stuck as `submitted`.

* Linting

* Comments for potential enhancements to management command

* Enhance management command

- Replaced logs with print statements to avoid bloating NR.
- Now checking for validation errors (similar to cross-validation step) before proceeding to attempt dissemination.
- New viewflow routine for transitioning a SAC from `submitted` to `auditee_certified`. This is only ran in the event there are errors with validation OR dissemination.

* Fix test - submission_status_transitions

* Linting

* Curation library and dissemination fixes

- Updated some error codes and logging more responses in `move_to_disseminated`.
- Brought in a new application for enabling/disabling audit tracking from `jadudm/curation-api`, which creates a table `curation.record_version` as well as functions to manipulate it, for tracking the changelog of SACs.
- Now tracking the changelog of the SAC when running `move_to_dissemination`.
- Added a condition in `remove_workbook_artifacts.py` when logging after bulk deletion of S3 files to ensure the key is not null.
- Added a new routine in `IntakeToDissemination` which takes a status and finds the first occurring date of that status for a SAC. `fac_acceptance_date` moving forward will now be based off of the first time a SAC was `submitted`, rather than the most recent time a SAC was `submitted`.

* Feedback from jadud

- New class `CurationTracking`, which allows us to wrap the audit trailing using a `with`. See the changelog below for reference.
- Tracking `sac_reverted_from_submitted` in our curation table.
- Now using `else` in dissemination validation check.
- Calling `--disable` on `curation_audit_tracking` on startup of the application.

* Update curation_audit_tracking_init.sh

* Attempted fix - disable audit tracking on startup fails

* Move curation after migrations

* Update init_curation_auditing.sql

backend/audit/intake_to_dissemination.py

@@ -246,7 +246,14 @@ def load_passthrough(self):
         self.loaded_objects["Passthroughs"] = pass_objects
         return pass_objects
 
-    def _get_dates_from_sac(self):
+    def _get_first_date_by_status_from_sac(self, status):
+        sac = self.single_audit_checklist
+        for i in range(len(sac.transition_name)):
+            if sac.transition_name[i] == status:
+                return sac.transition_date[i]
+        raise ValueError("This SAC does not have the requested status.")
+
+    def _get_most_recent_dates_from_sac(self):
         return_dict = dict()
         sac = self.single_audit_checklist
         for status_choice in sac.STATUS_CHOICES:
@@ -283,12 +290,12 @@ def load_general(self):
         cognizant_agency = self.single_audit_checklist.cognizant_agency
         oversight_agency = self.single_audit_checklist.oversight_agency
 
-        dates_by_status = self._get_dates_from_sac()
+        dates_by_status = self._get_most_recent_dates_from_sac()
         status = self.single_audit_checklist.get_statuses()
         ready_for_certification_date = dates_by_status[status.READY_FOR_CERTIFICATION]
         if self.mode == IntakeToDissemination.DISSEMINATION:
             submitted_date = self._convert_utc_to_american_samoa_zone(
-                dates_by_status[status.SUBMITTED]
+                self._get_first_date_by_status_from_sac(status.SUBMITTED)
             )
             fac_accepted_date = submitted_date
             auditee_certify_name = auditee_certification["auditee_signature"][

backend/audit/management/commands/move_to_disseminated.py

@@ -0,0 +1,101 @@
+"""
+Allows the manual transition of an audit to dissemination.
+"""
+
+from audit.models import (
+    SingleAuditChecklist,
+)
+from audit.models.models import STATUS
+from audit.models.viewflow import sac_revert_from_submitted, sac_transition
+from curation.curationlib.curation_audit_tracking import CurationTracking
+from dissemination.remove_workbook_artifacts import remove_workbook_artifacts
+from django.core.management.base import BaseCommand
+from django.db import transaction
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Django management command for disseminating an audit.
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--report_id",
+            type=str,
+            help="The ID of the SAC.",
+            default=None,
+        )
+
+    def handle(self, *args, **options):
+        report_id = options.get("report_id")
+
+        # no parameter passed.
+        if report_id is None:
+            logger.info(
+                "No report_id supplied. (move_to_disseminated --report_id ID_OF_REPORT)"
+            )
+            exit(-1)
+
+        try:
+            sac = SingleAuditChecklist.objects.get(report_id=report_id)
+        except SingleAuditChecklist.DoesNotExist:
+            logger.info(f"No report with report_id found: {report_id}")
+            exit(-1)
+
+        # must be stuck as 'submitted'.
+        if sac.submission_status != STATUS.SUBMITTED:
+            logger.info(
+                f"Unable to disseminate report that is not in submitted state: {report_id}"
+            )
+            exit(-1)
+
+        # check for validation errors.
+        errors = sac.validate_full()
+        if errors:
+            logger.info(
+                f"Unable to disseminate report with validation errors: {report_id}."
+            )
+            logger.info(errors["errors"])
+
+            # return to auditee_certified.
+            sac_revert_from_submitted(sac)
+            logger.info(f"Returned report to auditee_certified state: {report_id}")
+            exit(0)
+
+        with CurationTracking():
+            # BEGIN ATOMIC BLOCK
+            with transaction.atomic():
+                disseminated = sac.disseminate()
+                # `disseminated` is None if there were no errors.
+                if disseminated is None:
+                    sac_transition(None, sac, transition_to=STATUS.DISSEMINATED)
+            # END ATOMIC BLOCK
+
+        # IF THE DISSEMINATION SUCCEEDED
+        # `disseminated` is None if there were no errors.
+        if disseminated is None:
+            # Remove workbook artifacts after the report has been disseminated.
+            # We do this outside of the atomic block. No race between
+            # two instances of the FAC should be able to get to this point.
+            # If we do, something will fail.
+            remove_workbook_artifacts(sac)
+
+        # IF THE DISSEMINATION FAILED
+        # If disseminated has a value, it is an error
+        # object returned from `sac.disseminate()`
+        else:
+            logger.info(
+                "{} is a `not None` value report_id[{}] for `disseminated`".format(
+                    disseminated, report_id
+                )
+            )
+
+            # return to auditee_certified.
+            sac_revert_from_submitted(sac)
+            exit(0)
+
+        logger.info(f"DISSEMINATED REPORT: {report_id}")
+        exit(0)

backend/audit/models/viewflow.py

@@ -1,12 +1,34 @@
 from audit.models import SingleAuditChecklist, SubmissionEvent
 from audit.models.models import STATUS
+from curation.curationlib.curation_audit_tracking import CurationTracking
 import datetime
 import logging
 import viewflow.fsm
 
 logger = logging.getLogger(__name__)
 
 
+def sac_revert_from_submitted(sac):
+    """
+    Transitions the submission_state for a SingleAuditChecklist back
+    to "auditee_certified" so the user can re-address issues and submit.
+    This should only be executed via management command.
+    """
+
+    if sac.submission_status == STATUS.SUBMITTED:
+        flow = SingleAuditChecklistFlow(sac)
+
+        flow.transition_to_auditee_certified()
+
+        with CurationTracking():
+            sac.save(
+                event_user=None,
+                event_type=SubmissionEvent.EventType.AUDITEE_CERTIFICATION_COMPLETED,
+            )
+        return True
+    return False
+
+
 def sac_transition(request, sac, **kwargs):
     """
     Transitions the submission_state for a SingleAuditChecklist (sac).
@@ -139,7 +161,7 @@ def transition_to_auditor_certified(self):
         self.sac.transition_date.append(datetime.datetime.now(datetime.timezone.utc))
 
     @state.transition(
-        source=STATUS.AUDITOR_CERTIFIED,
+        source=[STATUS.AUDITOR_CERTIFIED, STATUS.SUBMITTED],
         target=STATUS.AUDITEE_CERTIFIED,
     )
     def transition_to_auditee_certified(self):
@@ -163,6 +185,8 @@ def transition_to_submitted(self):
         self.sac.transition_name.append(STATUS.SUBMITTED)
         self.sac.transition_date.append(datetime.datetime.now(datetime.timezone.utc))
 
+    # WIP
+    # to add - source=[STATUS.SUBMITTED, STATUS.DISSEMINATED]
     @state.transition(
         source=STATUS.SUBMITTED,
         target=STATUS.DISSEMINATED,

backend/audit/test_models.py

@@ -76,7 +76,7 @@ def test_submission_status_transitions(self):
                 "transition_to_auditor_certified",
             ),
             (
-                [STATUS.AUDITOR_CERTIFIED],
+                [STATUS.AUDITOR_CERTIFIED, STATUS.SUBMITTED],
                 STATUS.AUDITEE_CERTIFIED,
                 "transition_to_auditee_certified",
             ),

backend/config/settings.py

@@ -130,6 +130,7 @@
     "dissemination",
     "census_historical_migration",
     "support",
+    "curation",
 ]
 
 MIDDLEWARE = [

backend/curation/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class CurationConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "curation"

backend/curation/curationlib/curation_audit_tracking.py

@@ -0,0 +1,29 @@
+from dissemination import api_versions
+
+SQL_PATH = "curation/sql"
+
+
+class CurationTracking:
+    """
+    Wraps "audit-tracking" around any block of logic.
+    This guarantees that any DB writes within the block will be recorded.
+    """
+
+    def __enter__(self):
+        enable_audit_curation()
+        return None
+
+    def __exit__(self, exc_type, exc_value, tb):
+        disable_audit_curation()
+
+
+def init_audit_curation():
+    api_versions.exec_sql_at_path(SQL_PATH, "init_curation_auditing.sql")
+
+
+def enable_audit_curation():
+    api_versions.exec_sql_at_path(SQL_PATH, "enable_curation_auditing.sql")
+
+
+def disable_audit_curation():
+    api_versions.exec_sql_at_path(SQL_PATH, "disable_curation_auditing.sql")

backend/curation/management/commands/curation_audit_tracking.py

@@ -0,0 +1,28 @@
+from django.core.management.base import BaseCommand
+
+# These are pulled from a library so they can be used
+# elsewhere in the code.
+from curation.curationlib.curation_audit_tracking import (
+    init_audit_curation,
+    enable_audit_curation,
+    disable_audit_curation,
+)
+
+
+class Command(BaseCommand):
+    help = """
+    Runs sql scripts  to recreate access tables for the postgrest API.
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument("-i", "--init", action="store_true", default=False)
+        parser.add_argument("-e", "--enable", action="store_true", default=False)
+        parser.add_argument("-d", "--disable", action="store_true", default=False)
+
+    def handle(self, *args, **options):
+        if options["init"]:
+            init_audit_curation()
+        elif options["enable"]:
+            enable_audit_curation()
+        elif options["disable"]:
+            disable_audit_curation()

backend/curation/sql/disable_curation_auditing.sql

@@ -0,0 +1,2 @@
+select curation.disable_tracking('public.audit_singleauditchecklist'::regclass);
+select curation.disable_tracking('public.support_cognizantassignment'::regclass);

backend/curation/sql/enable_curation_auditing.sql

@@ -0,0 +1,2 @@
+select curation.enable_tracking('public.audit_singleauditchecklist'::regclass);
+select curation.enable_tracking('public.support_cognizantassignment'::regclass);