From f6040d042c2c97df0f02e019fa1dce36c44dbf79 Mon Sep 17 00:00:00 2001 From: Dan Swick <2365503+danswick@users.noreply.github.com> Date: Thu, 9 Nov 2023 08:31:33 -0800 Subject: [PATCH 1/5] Adjust user setting when generating workbooks (#2583) * don't try to create a fake user * we do need a user afterall * linter --- .../commands/end_to_end_test_data_generator.py | 11 ++++++----- .../dissemination/workbooklib/workbook_creation.py | 8 ++++++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/backend/dissemination/management/commands/end_to_end_test_data_generator.py b/backend/dissemination/management/commands/end_to_end_test_data_generator.py index be5914be46..243496b7c1 100644 --- a/backend/dissemination/management/commands/end_to_end_test_data_generator.py +++ b/backend/dissemination/management/commands/end_to_end_test_data_generator.py @@ -12,13 +12,16 @@ class Command(BaseCommand): def add_arguments(self, parser): - parser.add_argument("--email", type=str, required=False) + parser.add_argument( + "--email", type=str, required=False, default=CYPRESS_TEST_EMAIL_ADDR + ) parser.add_argument("--dbkeys", type=str, required=False, default="") parser.add_argument("--years", type=str, required=False, default="") def handle(self, *args, **options): dbkeys_str = options["dbkeys"] years_str = options["years"] + email_str = options["email"] dbkeys = dbkeys_str.split(",") years = years_str.split(",") @@ -35,8 +38,6 @@ def handle(self, *args, **options): logger.error("Years must be two digits. Exiting.") sys.exit(-2) - email = options.get("email", CYPRESS_TEST_EMAIL_ADDR) - defaults = [ (182926, 22), (181744, 22), @@ -49,11 +50,11 @@ def handle(self, *args, **options): f"Generating test reports for DBKEYS: {dbkeys_str} and YEARS: {years_str}" ) for dbkey, year in zip(dbkeys, years): - run_end_to_end(email, dbkey, year) + run_end_to_end(email_str, dbkey, year) else: for pair in defaults: logger.info("Running {}-{} end-to-end".format(pair[0], pair[1])) - run_end_to_end(email, str(pair[0]), str(pair[1])) + run_end_to_end(email_str, str(pair[0]), str(pair[1])) else: logger.error( "Cannot run end-to-end workbook generation in production. Exiting." diff --git a/backend/dissemination/workbooklib/workbook_creation.py b/backend/dissemination/workbooklib/workbook_creation.py index 8c7a539353..580eacfa3e 100644 --- a/backend/dissemination/workbooklib/workbook_creation.py +++ b/backend/dissemination/workbooklib/workbook_creation.py @@ -20,7 +20,6 @@ from dissemination.workbooklib.additional_eins import generate_additional_eins from dissemination.workbooklib.secondary_auditors import generate_secondary_auditors -from model_bakery import baker from django.contrib.auth import get_user_model import logging @@ -61,7 +60,12 @@ def setup_sac(user, test_name, dbkey): else: sac = SingleAuditChecklist() User = get_user_model() - user = baker.make(User) + email = "test_data@test.data" + User.objects.get_or_create( + username="test_data", + email=email, + ) + user = User.objects.get(email=email) sac.submitted_by = user sac.general_information = {} sac.general_information["auditee_name"] = test_name From 7854e9d5c7fd3436648591777de5b7049a2ebca0 Mon Sep 17 00:00:00 2001 From: Tadhg O'Higgins <2626258+tadhg-ohiggins@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:49:30 -0800 Subject: [PATCH 2/5] Move modellib to models, move models.py into models, move Access into its own file. (#2776) --- backend/audit/modellib/__init__.py | 8 -- backend/audit/models/__init__.py | 32 ++++++ backend/audit/models/access.py | 97 +++++++++++++++++++ backend/audit/{ => models}/models.py | 95 ++---------------- .../{modellib => models}/submission_event.py | 0 5 files changed, 136 insertions(+), 96 deletions(-) delete mode 100644 backend/audit/modellib/__init__.py create mode 100644 backend/audit/models/__init__.py create mode 100644 backend/audit/models/access.py rename backend/audit/{ => models}/models.py (89%) rename backend/audit/{modellib => models}/submission_event.py (100%) diff --git a/backend/audit/modellib/__init__.py b/backend/audit/modellib/__init__.py deleted file mode 100644 index dc269d13e9..0000000000 --- a/backend/audit/modellib/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from .submission_event import ( # noqa - SubmissionEvent, -) - -# In case we want to iterate through all the models for some reason: -models = [ - SubmissionEvent, -] diff --git a/backend/audit/models/__init__.py b/backend/audit/models/__init__.py new file mode 100644 index 0000000000..0a6a4cc79b --- /dev/null +++ b/backend/audit/models/__init__.py @@ -0,0 +1,32 @@ +from .access import Access +from .models import ( + ExcelFile, + GeneralInformationMixin, + LateChangeError, + SingleAuditChecklist, + SingleAuditChecklistManager, + SingleAuditReportFile, + User, + excel_file_path, + generate_sac_report_id, + single_audit_report_path, +) +from .submission_event import SubmissionEvent + +# In case we want to iterate through all the models for some reason: +_models = [ + Access, + ExcelFile, + GeneralInformationMixin, + SubmissionEvent, + LateChangeError, + SingleAuditChecklist, + SingleAuditChecklistManager, + SingleAuditReportFile, + User, +] +_functions = [ + excel_file_path, + generate_sac_report_id, + single_audit_report_path, +] diff --git a/backend/audit/models/access.py b/backend/audit/models/access.py new file mode 100644 index 0000000000..2aeff7fddd --- /dev/null +++ b/backend/audit/models/access.py @@ -0,0 +1,97 @@ +from django.contrib.auth import get_user_model +from django.db import models +from django.db.models import Q +from django.utils.translation import gettext_lazy as _ +from .models import ( + SingleAuditChecklist, + SubmissionEvent, +) + + +User = get_user_model() + + +class AccessManager(models.Manager): + """Custom manager for Access.""" + + def create(self, **obj_data): + """ + Check for existing users and add them at access creation time. + Not doing this would mean that users logged in at time of Access + instance creation would have to log out and in again to get the new + access. + """ + + # remove event_user & event_type keys so that they're not passed into super().create below + event_user = obj_data.pop("event_user", None) + event_type = obj_data.pop("event_type", None) + + if obj_data["email"]: + try: + acc_user = User.objects.get(email=obj_data["email"]) + except User.DoesNotExist: + acc_user = None + if acc_user: + obj_data["user"] = acc_user + result = super().create(**obj_data) + + if event_user and event_type: + SubmissionEvent.objects.create( + sac=result.sac, + user=event_user, + event=event_type, + ) + + return result + + +class Access(models.Model): + """ + Email addresses which have been granted access to SAC instances. + An email address may be associated with a User ID if an FAC account exists. + """ + + objects = AccessManager() + + ROLES = ( + ("certifying_auditee_contact", _("Auditee Certifying Official")), + ("certifying_auditor_contact", _("Auditor Certifying Official")), + ("editor", _("Audit Editor")), + ) + sac = models.ForeignKey(SingleAuditChecklist, on_delete=models.CASCADE) + role = models.CharField( + choices=ROLES, + help_text="Access type granted to this user", + max_length=50, + ) + fullname = models.CharField(blank=True) + email = models.EmailField() + user = models.ForeignKey( + User, + null=True, + help_text="User ID associated with this email address, empty if no FAC account exists", + on_delete=models.PROTECT, + ) + + def __str__(self): + return f"{self.email} as {self.get_role_display()}" + + class Meta: + """Constraints for certifying roles""" + + verbose_name_plural = "accesses" + + constraints = [ + # a SAC cannot have multiple certifying auditees + models.UniqueConstraint( + fields=["sac"], + condition=Q(role="certifying_auditee_contact"), + name="%(app_label)s_$(class)s_single_certifying_auditee", + ), + # a SAC cannot have multiple certifying auditors + models.UniqueConstraint( + fields=["sac"], + condition=Q(role="certifying_auditor_contact"), + name="%(app_label)s_%(class)s_single_certifying_auditor", + ), + ] diff --git a/backend/audit/models.py b/backend/audit/models/models.py similarity index 89% rename from backend/audit/models.py rename to backend/audit/models/models.py index 251d29a4bc..e8ced411f1 100644 --- a/backend/audit/models.py +++ b/backend/audit/models/models.py @@ -4,7 +4,6 @@ import logging from django.db import models -from django.db.models import Q from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.postgres.fields import ArrayField @@ -15,7 +14,6 @@ import audit.cross_validation from audit.intake_to_dissemination import IntakeToDissemination -from audit.modellib import SubmissionEvent from audit.validators import ( validate_additional_ueis_json, validate_additional_eins_json, @@ -35,6 +33,7 @@ validate_component_page_numbers, ) from support.cog_over import compute_cog_over, record_cog_assignment +from .submission_event import SubmissionEvent User = get_user_model() @@ -147,7 +146,10 @@ def inner(self): class LateChangeError(Exception): - pass + """ + Exception covering attempts to change submissions that don't have the in_progress + status. + """ class SingleAuditChecklist(models.Model, GeneralInformationMixin): # type: ignore @@ -586,90 +588,6 @@ def get_transition_date(self, status): return None -class AccessManager(models.Manager): - """Custom manager for Access.""" - - def create(self, **obj_data): - """ - Check for existing users and add them at access creation time. - Not doing this would mean that users logged in at time of Access - instance creation would have to log out and in again to get the new - access. - """ - - # remove event_user & event_type keys so that they're not passed into super().create below - event_user = obj_data.pop("event_user", None) - event_type = obj_data.pop("event_type", None) - - if obj_data["email"]: - try: - acc_user = User.objects.get(email=obj_data["email"]) - except User.DoesNotExist: - acc_user = None - if acc_user: - obj_data["user"] = acc_user - result = super().create(**obj_data) - - if event_user and event_type: - SubmissionEvent.objects.create( - sac=result.sac, - user=event_user, - event=event_type, - ) - - return result - - -class Access(models.Model): - """ - Email addresses which have been granted access to SAC instances. - An email address may be associated with a User ID if an FAC account exists. - """ - - objects = AccessManager() - - ROLES = ( - ("certifying_auditee_contact", _("Auditee Certifying Official")), - ("certifying_auditor_contact", _("Auditor Certifying Official")), - ("editor", _("Audit Editor")), - ) - sac = models.ForeignKey(SingleAuditChecklist, on_delete=models.CASCADE) - role = models.CharField( - choices=ROLES, - help_text="Access type granted to this user", - max_length=50, - ) - fullname = models.CharField(blank=True) - email = models.EmailField() - user = models.ForeignKey( - User, - null=True, - help_text="User ID associated with this email address, empty if no FAC account exists", - on_delete=models.PROTECT, - ) - - def __str__(self): - return f"{self.email} as {self.get_role_display()}" - - class Meta: - verbose_name_plural = "accesses" - - constraints = [ - # a SAC cannot have multiple certifying auditees - models.UniqueConstraint( - fields=["sac"], - condition=Q(role="certifying_auditee_contact"), - name="%(app_label)s_$(class)s_single_certifying_auditee", - ), - # a SAC cannot have multiple certifying auditors - models.UniqueConstraint( - fields=["sac"], - condition=Q(role="certifying_auditor_contact"), - name="%(app_label)s_%(class)s_single_certifying_auditor", - ), - ] - - def excel_file_path(instance, _filename): """ We want the actual filename in the filesystem to be unique and determined @@ -719,7 +637,8 @@ def single_audit_report_path(instance, _filename): class SingleAuditReportFile(models.Model): """ - Data model to track uploaded Single Audit report PDFs and associate them with SingleAuditChecklists + Data model to track uploaded Single Audit report PDFs and associate them + with SingleAuditChecklists """ file = models.FileField( diff --git a/backend/audit/modellib/submission_event.py b/backend/audit/models/submission_event.py similarity index 100% rename from backend/audit/modellib/submission_event.py rename to backend/audit/models/submission_event.py From 6e80e8d13f103a9a75822ac4c8fd012fe89342ef Mon Sep 17 00:00:00 2001 From: Tadhg O'Higgins <2626258+tadhg-ohiggins@users.noreply.github.com> Date: Thu, 9 Nov 2023 13:53:11 -0800 Subject: [PATCH 3/5] Handle duplicate email address issues (#2784) * First pass at fixing duplicate email problem. * add test case for access reassignment * add negative assertions to test case --------- Co-authored-by: Tim Ballard <1425377+timoballard@users.noreply.github.com> --- backend/api/views.py | 10 ++++---- backend/users/auth.py | 13 ++++++---- backend/users/test_auth.py | 49 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 10 deletions(-) diff --git a/backend/api/views.py b/backend/api/views.py index 24ca4d45a1..7d3a787a15 100644 --- a/backend/api/views.py +++ b/backend/api/views.py @@ -123,7 +123,7 @@ def access_and_submission_check(user, data): Access.objects.create( sac=sac, role="editor", - email=user.email, + email=str(user.email).lower(), user=user, event_user=user, event_type=SubmissionEvent.EventType.ACCESS_GRANTED, @@ -132,7 +132,7 @@ def access_and_submission_check(user, data): sac=sac, role="certifying_auditee_contact", fullname=serializer.data.get("certifying_auditee_contact_fullname"), - email=serializer.data.get("certifying_auditee_contact_email"), + email=serializer.data.get("certifying_auditee_contact_email").lower(), event_user=user, event_type=SubmissionEvent.EventType.ACCESS_GRANTED, ) @@ -140,7 +140,7 @@ def access_and_submission_check(user, data): sac=sac, role="certifying_auditor_contact", fullname=serializer.data.get("certifying_auditor_contact_fullname"), - email=serializer.data.get("certifying_auditor_contact_email"), + email=serializer.data.get("certifying_auditor_contact_email").lower(), event_user=user, event_type=SubmissionEvent.EventType.ACCESS_GRANTED, ) @@ -160,7 +160,7 @@ def access_and_submission_check(user, data): sac=sac, role="editor", fullname=name, - email=email, + email=str(email).lower(), event_user=user, event_type=SubmissionEvent.EventType.ACCESS_GRANTED, ) @@ -169,7 +169,7 @@ def access_and_submission_check(user, data): sac=sac, role="editor", fullname=name, - email=email, + email=str(email).lower(), event_user=user, event_type=SubmissionEvent.EventType.ACCESS_GRANTED, ) diff --git a/backend/users/auth.py b/backend/users/auth.py index ed0332c8e7..90f6aca4d4 100644 --- a/backend/users/auth.py +++ b/backend/users/auth.py @@ -11,11 +11,14 @@ def claim_audit_access(user, all_emails): - for email in all_emails: - access_invites = ( - Access.objects.filter(user_id=None) - .filter(email__iexact=email) - .update(user_id=user.id) + """ + user is our system user + all_emails is the list of email addresses from the login.gov JWT. + """ + l_emails = [ea.lower() for ea in all_emails] + for email in l_emails: + access_invites = Access.objects.filter(email__iexact=email).update( + user_id=user.id, email=email ) logger.debug(f"{user.email} granted access to {access_invites} new audits") diff --git a/backend/users/test_auth.py b/backend/users/test_auth.py index 7d1b6fabad..734da37a6a 100644 --- a/backend/users/test_auth.py +++ b/backend/users/test_auth.py @@ -132,3 +132,52 @@ def test_multiple_audit_access_granted(self): self.assertEqual(user, updated_access1.user) self.assertEqual(user, updated_access2.user) + + def test_claimed_access_reassigned_to_new_user(self): + """ + If a user presents with an email that matches Access objects that have already been assigned to a different user, those Access objects should be updated to point to the new user + """ + backend = FACAuthenticationBackend() + + email = "a@a.com" + + login_id_1 = str(uuid4()) + login_id_2 = str(uuid4()) + + # given that we have an existing user (user_1) with email a@a.com + user_a_1 = baker.make(User, username=login_id_1, email=email) + + # and that user has some claimed Accesses + access_1 = baker.make(Access, email=email, user=user_a_1) + access_2 = baker.make(Access, email=email, user=user_a_1) + + # and there are other claimed Accesses for other users + user_b = baker.make(User, email="b@b.com") + access_3 = baker.make(Access, email="b@b.com", user=user_b) + + # and there are other unclaimed Accesses for other emails + access_4 = baker.make(Access, email="c@c.com", user=None) + + user_a_2_info = {"sub": login_id_2, "email": email, "all_emails": [email]} + + factory = RequestFactory() + request = factory.get("/") + + # when a different login.gov user (user_a_2) presents with email a@a.com + user_a_2 = backend.authenticate(request, **user_a_2_info) + + updated_access_1 = Access.objects.get(pk=access_1.id) + updated_access_2 = Access.objects.get(pk=access_2.id) + updated_access_3 = Access.objects.get(pk=access_3.id) + updated_access_4 = Access.objects.get(pk=access_4.id) + + # then the Access objects formerly associated with user_a_1 should now be associated with user_a_2 + self.assertEqual(user_a_2, updated_access_1.user) + self.assertEqual(user_a_2, updated_access_2.user) + + # and the Access object associated with user_3 remain unchanged + self.assertEqual(user_b, updated_access_3.user) + + # and the unclaimed Access object remains unclaimed + self.assertEqual("c@c.com", updated_access_4.email) + self.assertEqual(None, updated_access_4.user) From de03faedad041d9d7e9173378f01d1160140183e Mon Sep 17 00:00:00 2001 From: Tadhg O'Higgins <2626258+tadhg-ohiggins@users.noreply.github.com> Date: Thu, 9 Nov 2023 13:55:24 -0800 Subject: [PATCH 4/5] Make admin view for Access have read-only sac value (#2781) * Make admin view for Access have read-only sac value. * Make admin view for Access have read-only sac value. --- backend/audit/admin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/audit/admin.py b/backend/audit/admin.py index 3232bcc13c..4c1265776a 100644 --- a/backend/audit/admin.py +++ b/backend/audit/admin.py @@ -30,6 +30,7 @@ class AccessAdmin(admin.ModelAdmin): list_display = ("sac", "role", "email") list_filter = ["role"] + readonly_fields = ("sac",) class ExcelFileAdmin(admin.ModelAdmin): From 5a8ec7f8474f2b51623554d49a57ef957f0b1c3b Mon Sep 17 00:00:00 2001 From: Phil Dominguez <142051477+phildominguez-gsa@users.noreply.github.com> Date: Thu, 9 Nov 2023 18:16:03 -0500 Subject: [PATCH 5/5] #2676: `excel_creation.py` - Refactoring template names and paths (#2779) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Splitting out templates.py * Replacing name strings * Lint * Getting paths from excel.py --------- Co-authored-by: Phil Dominguez <“philip.dominguez@gsa.gov”> --- .../commands/historic_workbook_generator.py | 11 --------- .../workbooklib/additional_eins.py | 7 +++--- .../workbooklib/additional_ueis.py | 7 +++--- .../workbooklib/corrective_action_plan.py | 8 ++++--- .../workbooklib/excel_creation.py | 17 -------------- .../workbooklib/federal_awards.py | 9 ++++---- .../workbooklib/findings.py | 9 ++++---- .../workbooklib/findings_text.py | 6 ++--- .../workbooklib/notes_to_sefa.py | 9 ++++---- .../workbooklib/secondary_auditors.py | 10 ++++---- .../workbooklib/templates.py | 23 +++++++++++++++++++ 11 files changed, 56 insertions(+), 60 deletions(-) create mode 100644 backend/census_historical_migration/workbooklib/templates.py diff --git a/backend/census_historical_migration/management/commands/historic_workbook_generator.py b/backend/census_historical_migration/management/commands/historic_workbook_generator.py index 51b9004a4a..9916afb11e 100644 --- a/backend/census_historical_migration/management/commands/historic_workbook_generator.py +++ b/backend/census_historical_migration/management/commands/historic_workbook_generator.py @@ -36,17 +36,6 @@ # before filling in the XLSX workbooks. FieldMap = NT("FieldMap", "in_sheet in_db default type") -templates = { - "AdditionalUEIs": "additional-ueis-workbook.xlsx", - "AdditionalEINs": "additional-eins-workbook.xlsx", - "AuditFindingsText": "audit-findings-text-workbook.xlsx", - "CAP": "corrective-action-plan-workbook.xlsx", - "AuditFindings": "federal-awards-audit-findings-workbook.xlsx", - "FederalAwards": "federal-awards-workbook.xlsx", - "SEFA": "notes-to-sefa-workbook.xlsx", - "SecondaryAuditors": "secondary-auditors-workbook.xlsx", -} - def set_single_cell_range(wb, range_name, value): the_range = wb.defined_names[range_name] diff --git a/backend/census_historical_migration/workbooklib/additional_eins.py b/backend/census_historical_migration/workbooklib/additional_eins.py index 5f2300c2f7..e3be164a8d 100644 --- a/backend/census_historical_migration/workbooklib/additional_eins.py +++ b/backend/census_historical_migration/workbooklib/additional_eins.py @@ -1,14 +1,13 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, WorkbookFieldInDissem, - templates, set_uei, map_simple_columns, generate_dissemination_test_table, ) - - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.census_models.census import dynamic_import +from audit.fixtures.excel import FORM_SECTIONS import openpyxl as pyxl @@ -25,7 +24,7 @@ def generate_additional_eins(dbkey, year, outfile): logger.info(f"--- generate additional eins {dbkey} {year} ---") Gen = dynamic_import("Gen", year) Eins = dynamic_import("Eins", year) - wb = pyxl.load_workbook(templates["AdditionalEINs"]) + wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.ADDITIONAL_EINS]) g = set_uei(Gen, wb, dbkey) diff --git a/backend/census_historical_migration/workbooklib/additional_ueis.py b/backend/census_historical_migration/workbooklib/additional_ueis.py index 9666f990a9..bbeff52dfb 100644 --- a/backend/census_historical_migration/workbooklib/additional_ueis.py +++ b/backend/census_historical_migration/workbooklib/additional_ueis.py @@ -1,14 +1,13 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, WorkbookFieldInDissem, - templates, set_uei, map_simple_columns, generate_dissemination_test_table, ) - - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.census_models.census import dynamic_import +from audit.fixtures.excel import FORM_SECTIONS import openpyxl as pyxl @@ -25,7 +24,7 @@ def generate_additional_ueis(dbkey, year, outfile): logger.info(f"--- generate additional ueis {dbkey} {year} ---") Gen = dynamic_import("Gen", year) - wb = pyxl.load_workbook(templates["AdditionalUEIs"]) + wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.ADDITIONAL_UEIS]) g = set_uei(Gen, wb, dbkey) if int(year) >= 22: Ueis = dynamic_import("Ueis", year) diff --git a/backend/census_historical_migration/workbooklib/corrective_action_plan.py b/backend/census_historical_migration/workbooklib/corrective_action_plan.py index cdcbe5fb43..10d7dd04b2 100644 --- a/backend/census_historical_migration/workbooklib/corrective_action_plan.py +++ b/backend/census_historical_migration/workbooklib/corrective_action_plan.py @@ -1,14 +1,14 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, WorkbookFieldInDissem, - templates, set_uei, map_simple_columns, generate_dissemination_test_table, test_pfix, ) - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.census_models.census import dynamic_import +from audit.fixtures.excel import FORM_SECTIONS import openpyxl as pyxl @@ -22,7 +22,9 @@ def generate_corrective_action_plan(dbkey, year, outfile): logger.info(f"--- generate corrective action plan {dbkey} {year} ---") Gen = dynamic_import("Gen", year) Captext = dynamic_import("Captext", year) - wb = pyxl.load_workbook(templates["CAP"]) + wb = pyxl.load_workbook( + sections_to_template_paths[FORM_SECTIONS.CORRECTIVE_ACTION_PLAN] + ) mappings = [ FieldMap("reference_number", "findingrefnums", "finding_ref_number", None, str), FieldMap("planned_action", "text", WorkbookFieldInDissem, None, test_pfix(3)), diff --git a/backend/census_historical_migration/workbooklib/excel_creation.py b/backend/census_historical_migration/workbooklib/excel_creation.py index 6f3bd37a17..96cf329fb7 100644 --- a/backend/census_historical_migration/workbooklib/excel_creation.py +++ b/backend/census_historical_migration/workbooklib/excel_creation.py @@ -1,6 +1,5 @@ from collections import namedtuple as NT from playhouse.shortcuts import model_to_dict -import os import sys import logging @@ -17,22 +16,6 @@ FieldMap = NT("FieldMap", "in_sheet in_db in_dissem default type") WorkbookFieldInDissem = 1000 -templates_root = "schemas/output/excel/xlsx/" -templates_raw = { - "AdditionalUEIs": "additional-ueis-workbook.xlsx", - "AdditionalEINs": "additional-eins-workbook.xlsx", - "AuditFindingsText": "audit-findings-text-workbook.xlsx", - "CAP": "corrective-action-plan-workbook.xlsx", - "AuditFindings": "federal-awards-audit-findings-workbook.xlsx", - "FederalAwards": "federal-awards-workbook.xlsx", - "SEFA": "notes-to-sefa-workbook.xlsx", - "SecondaryAuditors": "secondary-auditors-workbook.xlsx", -} - -templates = {} -for k, v in templates_raw.items(): - templates[k] = os.path.join(templates_root, v) - def test_pfix(n): def _test(o): diff --git a/backend/census_historical_migration/workbooklib/federal_awards.py b/backend/census_historical_migration/workbooklib/federal_awards.py index 7d4497c6b7..e13aebeb5b 100644 --- a/backend/census_historical_migration/workbooklib/federal_awards.py +++ b/backend/census_historical_migration/workbooklib/federal_awards.py @@ -1,16 +1,15 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, WorkbookFieldInDissem, - templates, set_uei, set_single_cell_range, map_simple_columns, generate_dissemination_test_table, set_range, ) - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.census_models.census import dynamic_import - +from audit.fixtures.excel import FORM_SECTIONS from config import settings import openpyxl as pyxl @@ -197,7 +196,9 @@ def generate_federal_awards(dbkey, year, outfile): Gen = dynamic_import("Gen", year) Passthrough = dynamic_import("Passthrough", year) Cfda = dynamic_import("Cfda", year) - wb = pyxl.load_workbook(templates["FederalAwards"]) + wb = pyxl.load_workbook( + sections_to_template_paths[FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED] + ) # In sheet : in DB g = set_uei(Gen, wb, dbkey) diff --git a/backend/census_historical_migration/workbooklib/findings.py b/backend/census_historical_migration/workbooklib/findings.py index dc209ab769..c9d097c29d 100644 --- a/backend/census_historical_migration/workbooklib/findings.py +++ b/backend/census_historical_migration/workbooklib/findings.py @@ -1,14 +1,13 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, - templates, set_uei, map_simple_columns, generate_dissemination_test_table, set_range, ) - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.census_models.census import dynamic_import - +from audit.fixtures.excel import FORM_SECTIONS import openpyxl as pyxl @@ -93,7 +92,9 @@ def generate_findings(dbkey, year, outfile): Gen = dynamic_import("Gen", year) Findings = dynamic_import("Findings", year) Cfda = dynamic_import("Cfda", year) - wb = pyxl.load_workbook(templates["AuditFindings"]) + wb = pyxl.load_workbook( + sections_to_template_paths[FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE] + ) g = set_uei(Gen, wb, dbkey) cfdas = Cfda.select().where(Cfda.dbkey == g.dbkey).order_by(Cfda.index) diff --git a/backend/census_historical_migration/workbooklib/findings_text.py b/backend/census_historical_migration/workbooklib/findings_text.py index d06269339d..610ea06b43 100644 --- a/backend/census_historical_migration/workbooklib/findings_text.py +++ b/backend/census_historical_migration/workbooklib/findings_text.py @@ -1,14 +1,14 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, WorkbookFieldInDissem, - templates, set_uei, map_simple_columns, generate_dissemination_test_table, test_pfix, ) - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.census_models.census import dynamic_import +from audit.fixtures.excel import FORM_SECTIONS import openpyxl as pyxl @@ -29,7 +29,7 @@ def generate_findings_text(dbkey, year, outfile): logger.info(f"--- generate findings text {dbkey} {year} ---") Gen = dynamic_import("Gen", year) Findingstext = dynamic_import("Findingstext", year) - wb = pyxl.load_workbook(templates["AuditFindingsText"]) + wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.FINDINGS_TEXT]) g = set_uei(Gen, wb, dbkey) diff --git a/backend/census_historical_migration/workbooklib/notes_to_sefa.py b/backend/census_historical_migration/workbooklib/notes_to_sefa.py index cd713fec10..105b1e29df 100644 --- a/backend/census_historical_migration/workbooklib/notes_to_sefa.py +++ b/backend/census_historical_migration/workbooklib/notes_to_sefa.py @@ -1,24 +1,23 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, - templates, set_uei, set_single_cell_range, map_simple_columns, generate_dissemination_test_table, test_pfix, ) - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.excel_creation import ( set_range, ) from census_historical_migration.workbooklib.census_models.census import dynamic_import - +from audit.fixtures.excel import FORM_SECTIONS import openpyxl as pyxl + import re import logging -# import unidecode logger = logging.getLogger(__name__) @@ -43,7 +42,7 @@ def generate_notes_to_sefa(dbkey, year, outfile): logger.info(f"--- generate notes to sefa {dbkey} {year}---") Gen = dynamic_import("Gen", year) Notes = dynamic_import("Notes", year) - wb = pyxl.load_workbook(templates["SEFA"]) + wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.NOTES_TO_SEFA]) g = set_uei(Gen, wb, dbkey) diff --git a/backend/census_historical_migration/workbooklib/secondary_auditors.py b/backend/census_historical_migration/workbooklib/secondary_auditors.py index 4a956427c2..6354b315fa 100644 --- a/backend/census_historical_migration/workbooklib/secondary_auditors.py +++ b/backend/census_historical_migration/workbooklib/secondary_auditors.py @@ -1,16 +1,14 @@ from census_historical_migration.workbooklib.excel_creation import ( FieldMap, - templates, set_uei, map_simple_columns, generate_dissemination_test_table, test_pfix, ) - +from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.workbooklib.census_models.census import dynamic_import - - from census_historical_migration.workbooklib.sac_creation import add_hyphen_to_zip +from audit.fixtures.excel import FORM_SECTIONS import openpyxl as pyxl @@ -54,7 +52,9 @@ def generate_secondary_auditors(dbkey, year, outfile): logger.info(f"--- generate secondary auditors {dbkey} {year} ---") Gen = dynamic_import("Gen", year) Cpas = dynamic_import("Cpas", year) - wb = pyxl.load_workbook(templates["SecondaryAuditors"]) + wb = pyxl.load_workbook( + sections_to_template_paths[FORM_SECTIONS.SECONDARY_AUDITORS] + ) g = set_uei(Gen, wb, dbkey) diff --git a/backend/census_historical_migration/workbooklib/templates.py b/backend/census_historical_migration/workbooklib/templates.py new file mode 100644 index 0000000000..8202c15ed8 --- /dev/null +++ b/backend/census_historical_migration/workbooklib/templates.py @@ -0,0 +1,23 @@ +from audit.fixtures.excel import FORM_SECTIONS +from audit.fixtures.excel import ( + ADDITIONAL_EINS_TEMPLATE, + ADDITIONAL_UEIS_TEMPLATE, + CORRECTIVE_ACTION_PLAN_TEMPLATE, + FEDERAL_AWARDS_TEMPLATE, + FINDINGS_TEXT_TEMPLATE, + FINDINGS_UNIFORM_GUIDANCE_TEMPLATE, + NOTES_TO_SEFA_TEMPLATE, + SECONDARY_AUDITORS_TEMPLATE, +) + + +sections_to_template_paths = { + FORM_SECTIONS.ADDITIONAL_EINS: ADDITIONAL_EINS_TEMPLATE, + FORM_SECTIONS.ADDITIONAL_UEIS: ADDITIONAL_UEIS_TEMPLATE, + FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: CORRECTIVE_ACTION_PLAN_TEMPLATE, + FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: FEDERAL_AWARDS_TEMPLATE, + FORM_SECTIONS.FINDINGS_TEXT: FINDINGS_TEXT_TEMPLATE, + FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: FINDINGS_UNIFORM_GUIDANCE_TEMPLATE, + FORM_SECTIONS.NOTES_TO_SEFA: NOTES_TO_SEFA_TEMPLATE, + FORM_SECTIONS.SECONDARY_AUDITORS: SECONDARY_AUDITORS_TEMPLATE, +}