Skip to content

Commit

Permalink
Remove report flag for removal (#4566)
Browse files Browse the repository at this point in the history
* #4434 Added django admin action to delete reports flagged for removal for over 6 months

* Added test cases

* Captured removal flag event

* Adding transaction
  • Loading branch information
sambodeme authored Jan 2, 2025
1 parent b11e4eb commit 4692464
Showing 6 changed files with 245 additions and 4 deletions.
100 changes: 99 additions & 1 deletion backend/audit/admin.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import datetime
import logging
from django.conf import settings
from django.contrib import admin, messages
from django.shortcuts import redirect
from django.urls import reverse
from audit.forms import SacValidationWaiverForm, UeiValidationWaiverForm
from audit.models import (
Access,
@@ -24,9 +27,85 @@
)
from django.contrib.admin import SimpleListFilter
from django.utils.translation import gettext_lazy as _
from django.db import transaction
from django.utils.timezone import now

from dissemination.remove_singleauditreport_pdf import remove_singleauditreport_pdf
from dissemination.remove_workbook_artifacts import remove_workbook_artifacts

logger = logging.getLogger(__name__)

# As per ADR #0041, the retention period for flagged reports is 6 months. That is 180 days.
FLAGGED_REPORT_RETENTION_DAYS = 180 # 6 months


@admin.action(description="Delete reports flagged for removal for over 6 months")
def delete_flagged_records(modeladmin, request, queryset):
"""
Admin action to delete records flagged for removal older than the specified months.
"""
cutoff_date = now() - datetime.timedelta(days=FLAGGED_REPORT_RETENTION_DAYS)

# Filter records for deletion
records_to_delete = queryset.filter(
submission_status=STATUS.FLAGGED_FOR_REMOVAL,
).exclude(
transition_name__isnull=True,
transition_date__isnull=True,
)

count = 0
failed_count = 0
report_ids = []
failed_report_ids = []
for sac in records_to_delete:
try:
if sac.transition_name[-1] == STATUS.FLAGGED_FOR_REMOVAL:
# Get the corresponding transition_date
transition_datetime = sac.transition_date[-1]

if transition_datetime <= cutoff_date:
with transaction.atomic():
auditee_uei = sac.general_information.get("auditee_uei")
if auditee_uei:
UeiValidationWaiver.objects.filter(uei=auditee_uei).delete()
SacValidationWaiver.objects.filter(report_id=sac).delete()
remove_singleauditreport_pdf(sac)
remove_workbook_artifacts(sac)
Access.objects.filter(sac=sac).delete()
DeletedAccess.objects.filter(sac=sac).delete()
SubmissionEvent.objects.filter(sac=sac).delete()
ExcelFile.objects.filter(sac=sac).delete()
SingleAuditReportFile.objects.filter(sac=sac).delete()
report_ids.append(sac.report_id)
sac.delete()
count += 1
except Exception as e:
logger.error(
f"Failed to delete sac report with ID {sac.report_id}: {str(e)}"
)
failed_count += 1
failed_report_ids.append(sac.report_id)
continue

if count:
logger.info(
f"Deleted {count} flagged records and their associated child records. Report IDs: {', '.join(report_ids)}"
)
modeladmin.message_user(
request,
f"Successfully deleted {count} flagged records and their associated child records. Report IDs: {', '.join(report_ids)}",
)
if failed_count:
logger.error(
f"Failed to delete {failed_count} flagged records. Report IDs: {', '.join(failed_report_ids)}"
)
modeladmin.message_user(
request,
f"Failed to delete {failed_count} flagged records. Report IDs: {', '.join(failed_report_ids)}",
level=messages.ERROR,
)


@admin.action(description="Revert selected report(s) to In Progress")
def revert_to_in_progress(modeladmin, request, queryset):
@@ -125,7 +204,26 @@ def has_view_permission(self, request, obj=None):
]
readonly_fields = ("submitted_by",)
search_fields = ("general_information__auditee_uei", "report_id")
actions = [revert_to_in_progress, flag_for_removal]
actions = [revert_to_in_progress, flag_for_removal, delete_flagged_records]

def changelist_view(self, request, extra_context=None):
"""
Override changelist_view to allow running the action without selection.
"""
if (
"action" in request.POST
and request.POST["action"] == "delete_flagged_records"
):
queryset = self.get_queryset(request)
delete_flagged_records(self, request, queryset)
# Redirect to avoid Django's "No items selected" error and ensure a valid response
return redirect(
reverse(
f"admin:{self.opts.app_label}_{self.opts.model_name}_changelist"
)
)

return super().changelist_view(request, extra_context=extra_context)


class AccessAdmin(admin.ModelAdmin):
76 changes: 75 additions & 1 deletion backend/audit/test_admin.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
from unittest.mock import MagicMock, patch
from django.test import TestCase, RequestFactory
from django.contrib.admin.sites import AdminSite
from django.contrib.auth.models import User
from django.contrib.messages.storage.fallback import FallbackStorage
from .models import SacValidationWaiver, SingleAuditChecklist
from .models.models import STATUS
from .models.models import STATUS, UeiValidationWaiver
from .admin import (
SACAdmin,
SacValidationWaiverAdmin,
delete_flagged_records,
flag_for_removal,
revert_to_in_progress,
)
@@ -226,3 +228,75 @@ def test_flag_for_removal_already_flagged(self):
self.assertEqual(self.report1.submission_status, STATUS.FLAGGED_FOR_REMOVAL)
messages = [m.message for m in self.request._messages]
self.assertIn("Report(s) (RPT001) were already flagged.", messages)


class TestDeleteFlaggedRecordsAdminAction(TestCase):

@patch("audit.admin.remove_workbook_artifacts")
@patch("audit.admin.remove_singleauditreport_pdf")
def test_delete_flagged_records_success(
self, mock_remove_workbook_artifacts, mock_remove_singleauditreport_pd
):
sac = baker.make(
SingleAuditChecklist,
report_id="1900-01-GSAFAC-0000000001",
submission_status=STATUS.FLAGGED_FOR_REMOVAL,
transition_name=["in_progress", STATUS.FLAGGED_FOR_REMOVAL],
transition_date=[
"2024-01-01 15:19:20.403935-05",
"2023-06-01 15:19:20.403935-05",
],
general_information={"auditee_uei": "ABCDHJH74DW7"},
)
baker.make(UeiValidationWaiver, uei="ABCDHJH74DW7")
baker.make(SacValidationWaiver, report_id=sac)

modeladmin = MagicMock()
queryset = SingleAuditChecklist.objects.filter(id=sac.id)

delete_flagged_records(modeladmin, None, queryset)

# Assert that related objects and SAC are deleted
self.assertFalse(
SingleAuditChecklist.objects.filter(
report_id="1900-01-GSAFAC-0000000001"
).exists()
)
self.assertFalse(
UeiValidationWaiver.objects.filter(uei="ABCDHJH74DW7").exists()
)
self.assertFalse(
SacValidationWaiver.objects.filter(
report_id="1900-01-GSAFAC-0000000001"
).exists()
)
mock_remove_singleauditreport_pd.assert_called_once()
mock_remove_workbook_artifacts.assert_called_once()

@patch("audit.admin.remove_workbook_artifacts")
@patch("audit.admin.remove_singleauditreport_pdf")
def test_delete_flagged_records_skips_non_flagged(
self, mock_remove_workbook_artifacts, mock_remove_singleauditreport_pd
):
sac = baker.make(
SingleAuditChecklist,
report_id="1900-01-GSAFAC-0000000002",
submission_status="in_progress",
transition_name=["in_progress"],
transition_date=["2024-01-01 15:19:20.403935-05"],
general_information={"auditee_uei": "EFJHHJH74DW7"},
)

modeladmin = MagicMock()
queryset = SingleAuditChecklist.objects.filter(id=sac.id)

delete_flagged_records(modeladmin, None, queryset)

# Assert that SAC is not deleted
self.assertTrue(
SingleAuditChecklist.objects.filter(
report_id="1900-01-GSAFAC-0000000002"
).exists()
)
mock_remove_workbook_artifacts.assert_not_called()
mock_remove_singleauditreport_pd.assert_not_called()
6 changes: 5 additions & 1 deletion backend/audit/views/remove_submission_in_progress.py
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@
from audit.views.views import verify_status
from audit.models.models import STATUS
from audit.models.viewflow import SingleAuditChecklistFlow
from audit.models.submission_event import SubmissionEvent

logger = logging.getLogger(__name__)

@@ -58,7 +59,10 @@ def post(self, request, *args, **kwargs):

flow = SingleAuditChecklistFlow(sac)
flow.transition_to_flagged_for_removal()
sac.save()
sac.save(
event_user=request.user,
event_type=SubmissionEvent.EventType.FLAGGED_SUBMISSION_FOR_REMOVAL,
)
url = reverse("audit:MySubmissions")

return redirect(url)
29 changes: 29 additions & 0 deletions backend/dissemination/remove_singleauditreport_pdf.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
import logging

from audit.models.models import SingleAuditReportFile

from dissemination.remove_workbook_artifacts import delete_files_in_bulk


logger = logging.getLogger(__name__)


def remove_singleauditreport_pdf(sac):
"""
Remove the single audit report pdf associated with the given sac.
"""
try:
files = SingleAuditReportFile.objects.filter(sac=sac)
files = [f"singleauditreport/{file.filename}" for file in files]
if files:
# Delete the files from S3 in bulk
delete_files_in_bulk(files, sac)

except SingleAuditReportFile.DoesNotExist:
logger.info(
f"No single audit report file found to delete for report: {sac.report_id}"
)
except Exception as e:
logger.error(
f"Failed to delete files from S3 for report: {sac.report_id}. Error: {e}"
)
6 changes: 5 additions & 1 deletion backend/dissemination/remove_workbook_artifacts.py
Original file line number Diff line number Diff line change
@@ -2,7 +2,11 @@
import math

from django.conf import settings
from audit.models.models import ExcelFile, SingleAuditChecklist, STATUS
from audit.models.models import (
ExcelFile,
SingleAuditChecklist,
STATUS,
)
from boto3 import client as boto3_client
from botocore.client import ClientError, Config
from django.core.paginator import Paginator
32 changes: 32 additions & 0 deletions backend/dissemination/test_remove_singleauditreport_pdf.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
from django.test import TestCase
from unittest.mock import patch
from audit.models.models import SingleAuditChecklist, SingleAuditReportFile
from dissemination.remove_singleauditreport_pdf import remove_singleauditreport_pdf
from model_bakery import baker


class TestRemoveSingleAuditReportPDF(TestCase):

@patch("dissemination.remove_singleauditreport_pdf.delete_files_in_bulk")
def test_remove_singleauditreport_pdf_success(self, mock_delete_files_in_bulk):
sac = baker.make(SingleAuditChecklist, report_id="1900-01-GSAFAC-0000000001")
baker.make(
SingleAuditReportFile,
filename="singleauditreport/1900-01-GSAFAC-0000000001.pdf",
sac=sac,
)

remove_singleauditreport_pdf(sac)

# Assert that delete_files_in_bulk is called with correct arguments
mock_delete_files_in_bulk.assert_called_once_with(
["singleauditreport/1900-01-GSAFAC-0000000001.pdf"], sac
)

@patch("dissemination.remove_singleauditreport_pdf.delete_files_in_bulk")
def test_remove_singleauditreport_pdf_no_files(self, mock_delete_files_in_bulk):
sac = baker.make(SingleAuditChecklist, report_id="1900-01-GSAFAC-0000000002")

remove_singleauditreport_pdf(sac)

mock_delete_files_in_bulk.assert_not_called()

0 comments on commit 4692464

Please sign in to comment.