From 385d662b90267dc195159208056406b75b20a68d Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Tue, 22 Oct 2024 13:51:59 -0400 Subject: [PATCH 01/19] WIP: Add yearly ts wrapper. --- .../hmda/analytics/HmdaAnalyticsApp.scala | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index bfcb29ca54..4a57722fa9 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -66,6 +66,35 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo * In beta namespace this environment variable has currentSchema=hmda_beta_user appended to it to change the schema * to BETA */ + + case class YearlyTransmittalSheetRepositoryWrapper(val year: String){ + val transmittalSheet = new TransmittalSheetRepository(dbConfig, getTableNameByYear(year)) + + def getTransmittalSheet = transmittalSheet + + def getId(row: TransmittalSheetTable): Unit = transmittalSheet.getId(row) + def createSchema(): Unit = transmittalSheet.createSchema() + def dropSchema(): Unit = transmittalSheet.dropSchema() + + def insert(ts: TransmittalSheetEntity): Future[Int] = transmittalSheet.insert(ts) + + def findByLei(lei: String): Future[Seq[TransmittalSheetEntity]] = transmittalSheet.findByLei(lei) + + def deleteByLei(lei: String): Future[Int] = transmittalSheet.deleteByLei(lei) + + def deleteByLeiAndQuarter(lei: String): Future[Int] = transmittalSheet.deleteByLeiAndQuarter(lei) + + def updateByLei(ts: TransmittalSheetEntity): Future[Int] = transmittalSheet.updateByLei(ts) + + def count(): Future[Int] = transmittalSheet.count() + + private def getTableNameByYear(year: String): String = { + val configString = s"hmda.analytics.$year.tsTableName" + config.getString(configString) + } + + } + val tsTableName2018 = config.getString("hmda.analytics.2018.tsTableName") val larTableName2018 = config.getString("hmda.analytics.2018.larTableName") //submission_history table remains same regardless of the year. There is a sign_date column and submission_id column which would show which year the filing was for From c8839569b0cd1ffa109defb06e68990eb0ef62b3 Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Tue, 22 Oct 2024 13:53:05 -0400 Subject: [PATCH 02/19] WIP: Add quarterly ts wrapper. --- .../hmda/analytics/HmdaAnalyticsApp.scala | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index 4a57722fa9..8dba918621 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -95,6 +95,35 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo } + case class QuarterlyTransmittalSheetRepositoryWrapper(val year: String, val quarter: String){ + val transmittalSheet = new TransmittalSheetRepository(dbConfig, getTableNameByYear(year)) + + def getTransmittalSheet = transmittalSheet + + + def getId(row: TransmittalSheetTable): Unit = transmittalSheet.getId(row) + def createSchema(): Unit = transmittalSheet.createSchema() + def dropSchema(): Unit = transmittalSheet.dropSchema() + + def insert(ts: TransmittalSheetEntity): Future[Int] = transmittalSheet.insert(ts) + + def findByLei(lei: String): Future[Seq[TransmittalSheetEntity]] = transmittalSheet.findByLei(lei) + + def deleteByLei(lei: String): Future[Int] = transmittalSheet.deleteByLei(lei) + + def deleteByLeiAndQuarter(lei: String): Future[Int] = transmittalSheet.deleteByLeiAndQuarter(lei) + + def updateByLei(ts: TransmittalSheetEntity): Future[Int] = transmittalSheet.updateByLei(ts) + + def count(): Future[Int] = transmittalSheet.count() + + private def getTableNameByYear(year: String): String = { + val configString = s"hmda.analytics.$year.tsTableName$quarter" + config.getString(configString) + } + + } + val tsTableName2018 = config.getString("hmda.analytics.2018.tsTableName") val larTableName2018 = config.getString("hmda.analytics.2018.larTableName") //submission_history table remains same regardless of the year. There is a sign_date column and submission_id column which would show which year the filing was for From 07eb74779abe54a145ce4ad893af1cb44ec6740f Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Tue, 22 Oct 2024 13:55:03 -0400 Subject: [PATCH 03/19] WIP: Replace delete ts row with wrapper. --- .../hmda/analytics/HmdaAnalyticsApp.scala | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index 8dba918621..4fdfeab341 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -285,27 +285,27 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo .mapAsync(1) { ts => for { delete <- submissionId.period match { - case Period(2018, None) => transmittalSheetRepository2018.deleteByLei(ts.lei) - case Period(2019, None) => transmittalSheetRepository2019.deleteByLei(ts.lei) - case Period(2020, Some("Q1")) => transmittalSheetRepository2020Q1.deleteByLeiAndQuarter(lei = ts.lei) - case Period(2020, Some("Q2")) => transmittalSheetRepository2020Q2.deleteByLeiAndQuarter(lei = ts.lei) - case Period(2020, Some("Q3")) => transmittalSheetRepository2020Q3.deleteByLeiAndQuarter(lei = ts.lei) - case Period(2020, None) => transmittalSheetRepository2020.deleteByLei(lei = ts.lei) - case Period(2021, None) => transmittalSheetRepository2021.deleteByLei(lei = ts.lei) - case Period(2022, None) => transmittalSheetRepository2022.deleteByLei(lei = ts.lei) - case Period(2023, None) => transmittalSheetRepository2023.deleteByLei(lei = ts.lei) - case Period(2021, Some("Q1")) => transmittalSheetRepository2021Q1.deleteByLeiAndQuarter(lei = ts.lei) - case Period(2021, Some("Q2")) => transmittalSheetRepository2021Q2.deleteByLeiAndQuarter(lei = ts.lei) - case Period(2021, Some("Q3")) => transmittalSheetRepository2021Q3.deleteByLeiAndQuarter(lei = ts.lei) - case Period(2022, Some("Q1")) => transmittalSheetRepository2022Q1.deleteByLeiAndQuarter(lei = ts.lei) - case Period(2022, Some("Q2")) => transmittalSheetRepository2022Q2.deleteByLeiAndQuarter(lei = ts.lei) - case Period(2022, Some("Q3")) => transmittalSheetRepository2022Q3.deleteByLeiAndQuarter(lei = ts.lei) - case Period(2023, Some("Q1")) => transmittalSheetRepository2023Q1.deleteByLeiAndQuarter(lei = ts.lei) - case Period(2023, Some("Q2")) => transmittalSheetRepository2023Q2.deleteByLeiAndQuarter(lei = ts.lei) - case Period(2023, Some("Q3")) => transmittalSheetRepository2023Q3.deleteByLeiAndQuarter(lei = ts.lei) - case Period(2024, Some("Q1")) => transmittalSheetRepository2024Q1.deleteByLeiAndQuarter(lei = ts.lei) - case Period(2024, Some("Q2")) => transmittalSheetRepository2024Q2.deleteByLeiAndQuarter(lei = ts.lei) - case Period(2024, Some("Q3")) => transmittalSheetRepository2024Q3.deleteByLeiAndQuarter(lei = ts.lei) + case Period(2018, None) => YearlyTransmittalSheetRepositoryWrapper("2018").deleteByLei(ts.lei) + case Period(2019, None) => YearlyTransmittalSheetRepositoryWrapper("2019").deleteByLei(ts.lei) + case Period(2020, Some("Q1")) => QuarterlyTransmittalSheetRepositoryWrapper("2020", "Q1").deleteByLei(ts.lei) + case Period(2020, Some("Q2")) => QuarterlyTransmittalSheetRepositoryWrapper("2020", "Q2").deleteByLei(ts.lei) + case Period(2020, Some("Q3")) => QuarterlyTransmittalSheetRepositoryWrapper("2020", "Q3").deleteByLei(ts.lei) + case Period(2020, None) => YearlyTransmittalSheetRepositoryWrapper("2020").deleteByLei(ts.lei) + case Period(2021, None) => YearlyTransmittalSheetRepositoryWrapper("2021").deleteByLei(ts.lei) + case Period(2022, None) => YearlyTransmittalSheetRepositoryWrapper("2022").deleteByLei(ts.lei) + case Period(2023, None) => YearlyTransmittalSheetRepositoryWrapper("2023").deleteByLei(ts.lei) + case Period(2021, Some("Q1")) => QuarterlyTransmittalSheetRepositoryWrapper("2021", "Q1").deleteByLei(ts.lei) + case Period(2021, Some("Q2")) => QuarterlyTransmittalSheetRepositoryWrapper("2021", "Q2").deleteByLei(ts.lei) + case Period(2021, Some("Q3")) => QuarterlyTransmittalSheetRepositoryWrapper("2021", "Q3").deleteByLei(ts.lei) + case Period(2022, Some("Q1")) => QuarterlyTransmittalSheetRepositoryWrapper("2022", "Q1").deleteByLei(ts.lei) + case Period(2022, Some("Q2")) => QuarterlyTransmittalSheetRepositoryWrapper("2022", "Q2").deleteByLei(ts.lei) + case Period(2022, Some("Q3")) => QuarterlyTransmittalSheetRepositoryWrapper("2022", "Q3").deleteByLei(ts.lei) + case Period(2023, Some("Q1")) => QuarterlyTransmittalSheetRepositoryWrapper("2023", "Q1").deleteByLei(ts.lei) + case Period(2023, Some("Q2")) => QuarterlyTransmittalSheetRepositoryWrapper("2023", "Q2").deleteByLei(ts.lei) + case Period(2023, Some("Q3")) => QuarterlyTransmittalSheetRepositoryWrapper("2023", "Q3").deleteByLei(ts.lei) + case Period(2024, Some("Q1")) => QuarterlyTransmittalSheetRepositoryWrapper("2024", "Q1").deleteByLei(ts.lei) + case Period(2024, Some("Q2")) => QuarterlyTransmittalSheetRepositoryWrapper("2024", "Q2").deleteByLei(ts.lei) + case Period(2024, Some("Q3")) => QuarterlyTransmittalSheetRepositoryWrapper("2024", "Q3").deleteByLei(ts.lei) case _ => { log.error(s"Unable to discern period from $submissionId to delete TS rows.") throw new IllegalArgumentException(s"Unable to discern period from $submissionId to delete TS rows.") From 7cb303efcdc6aae729243d393edf476d206ca5a5 Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Mon, 28 Oct 2024 12:27:10 -0400 Subject: [PATCH 04/19] WIP: Replace insert ts row with wrapper. --- .../hmda/analytics/HmdaAnalyticsApp.scala | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index 4fdfeab341..204e2f62e1 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -349,27 +349,27 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo .map(ts => TransmittalSheetConverter(ts, submissionIdOption)) .mapAsync(1) { ts => val (repo, enforceQuarterly) = submissionId.period match { - case Period(2018, None) => (transmittalSheetRepository2018, false) - case Period(2019, None) => (transmittalSheetRepository2019, false) - case Period(2020, Some("Q1")) => (transmittalSheetRepository2020Q1, true) - case Period(2020, Some("Q2")) => (transmittalSheetRepository2020Q2, true) - case Period(2020, Some("Q3")) => (transmittalSheetRepository2020Q3, true) - case Period(2020, None) => (transmittalSheetRepository2020, false) - case Period(2021, None) => (transmittalSheetRepository2021, false) - case Period(2022, None) => (transmittalSheetRepository2022, false) - case Period(2023, None) => (transmittalSheetRepository2023, false) - case Period(2021, Some("Q1")) => (transmittalSheetRepository2021Q1, true) - case Period(2021, Some("Q2")) => (transmittalSheetRepository2021Q2, true) - case Period(2021, Some("Q3")) => (transmittalSheetRepository2021Q3, true) - case Period(2022, Some("Q1")) => (transmittalSheetRepository2022Q1, true) - case Period(2022, Some("Q2")) => (transmittalSheetRepository2022Q2, true) - case Period(2022, Some("Q3")) => (transmittalSheetRepository2022Q3, true) - case Period(2023, Some("Q1")) => (transmittalSheetRepository2023Q1, true) - case Period(2023, Some("Q2")) => (transmittalSheetRepository2023Q2, true) - case Period(2023, Some("Q3")) => (transmittalSheetRepository2023Q3, true) - case Period(2024, Some("Q1")) => (transmittalSheetRepository2024Q1, true) - case Period(2024, Some("Q2")) => (transmittalSheetRepository2024Q2, true) - case Period(2024, Some("Q3")) => (transmittalSheetRepository2024Q3, true) + case Period(2018, None) => (YearlyTransmittalSheetRepositoryWrapper("2018").getTransmittalSheet, false) + case Period(2019, None) => (YearlyTransmittalSheetRepositoryWrapper("2019").getTransmittalSheet, false) + case Period(2020, Some("Q1")) => (QuarterlyTransmittalSheetRepositoryWrapper("2020", "Q1").getTransmittalSheet, true) + case Period(2020, Some("Q2")) => (QuarterlyTransmittalSheetRepositoryWrapper("2020", "Q2").getTransmittalSheet, true) + case Period(2020, Some("Q3")) => (QuarterlyTransmittalSheetRepositoryWrapper("2020", "Q3").getTransmittalSheet, true) + case Period(2020, None) => (YearlyTransmittalSheetRepositoryWrapper("2020").getTransmittalSheet, false) + case Period(2021, None) => (YearlyTransmittalSheetRepositoryWrapper("2021").getTransmittalSheet, false) + case Period(2022, None) => (YearlyTransmittalSheetRepositoryWrapper("2022").getTransmittalSheet, false) + case Period(2023, None) => (YearlyTransmittalSheetRepositoryWrapper("2023").getTransmittalSheet, false) + case Period(2021, Some("Q1")) => (QuarterlyTransmittalSheetRepositoryWrapper("2021", "Q1").getTransmittalSheet, true) + case Period(2021, Some("Q2")) => (QuarterlyTransmittalSheetRepositoryWrapper("2021", "Q2").getTransmittalSheet, true) + case Period(2021, Some("Q3")) => (QuarterlyTransmittalSheetRepositoryWrapper("2021", "Q3").getTransmittalSheet, true) + case Period(2022, Some("Q1")) => (QuarterlyTransmittalSheetRepositoryWrapper("2022", "Q1").getTransmittalSheet, true) + case Period(2022, Some("Q2")) => (QuarterlyTransmittalSheetRepositoryWrapper("2022", "Q2").getTransmittalSheet, true) + case Period(2022, Some("Q3")) => (QuarterlyTransmittalSheetRepositoryWrapper("2022", "Q3").getTransmittalSheet, true) + case Period(2023, Some("Q1")) => (QuarterlyTransmittalSheetRepositoryWrapper("2023", "Q1").getTransmittalSheet, true) + case Period(2023, Some("Q2")) => (QuarterlyTransmittalSheetRepositoryWrapper("2023", "Q2").getTransmittalSheet, true) + case Period(2023, Some("Q3")) => (QuarterlyTransmittalSheetRepositoryWrapper("2023", "Q3").getTransmittalSheet, true) + case Period(2024, Some("Q1")) => (QuarterlyTransmittalSheetRepositoryWrapper("2024", "Q1").getTransmittalSheet, true) + case Period(2024, Some("Q2")) => (QuarterlyTransmittalSheetRepositoryWrapper("2024", "Q2").getTransmittalSheet, true) + case Period(2024, Some("Q3")) => (QuarterlyTransmittalSheetRepositoryWrapper("2024", "Q3").getTransmittalSheet, true) case _ =>{ log.error(s"Unable to discern period from $submissionId to insert TS rows.") throw new IllegalArgumentException(s"Unable to discern period from $submissionId to insert TS rows.") From 3fcad828a1c34cee625b31112710ae7a5c01b33d Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Mon, 28 Oct 2024 12:57:56 -0400 Subject: [PATCH 05/19] Add Lar repo wrapper and delete unneeded values. --- .../hmda/analytics/HmdaAnalyticsApp.scala | 237 ++++++------------ 1 file changed, 83 insertions(+), 154 deletions(-) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index 204e2f62e1..fbd90f9b1d 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -67,7 +67,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo * to BETA */ - case class YearlyTransmittalSheetRepositoryWrapper(val year: String){ + case class YearlyTransmittalSheetRepositoryWrapper(year: String){ val transmittalSheet = new TransmittalSheetRepository(dbConfig, getTableNameByYear(year)) def getTransmittalSheet = transmittalSheet @@ -95,7 +95,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo } - case class QuarterlyTransmittalSheetRepositoryWrapper(val year: String, val quarter: String){ + case class QuarterlyTransmittalSheetRepositoryWrapper(year: String, quarter: String){ val transmittalSheet = new TransmittalSheetRepository(dbConfig, getTableNameByYear(year)) def getTransmittalSheet = transmittalSheet @@ -124,118 +124,47 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo } - val tsTableName2018 = config.getString("hmda.analytics.2018.tsTableName") - val larTableName2018 = config.getString("hmda.analytics.2018.larTableName") + case class YearlyLarRepositoryWrapper(year: String){ + val larRepo = new LarRepository(dbConfig, getTableNameByYear(year)) + + def getLarRepository = larRepo + + def insert(le: LarEntity): Future[Int] = larRepo.insert(le) + + private def getTableNameByYear(year: String): String = { + val configString = s"hmda.analytics.$year.larTableName" + config.getString(configString) + } + + } + + case class QuarterlyLarRepositoryWrapper(year: String, quarter: String){ + val larRepo = new LarRepository(dbConfig, getTableNameByYear(year)) + + def getLarRepository = larRepo + + // for 2019 and beyond + def insert(le: LarEntity): Future[Int] = larRepo.insert(le) + + // for 2018 + def insert(le: LarEntity2018): Future[Int] = larRepo.insert(le) + + def deletebyLeiAndQuarter(lei: String): Future[Int] = larRepo.deletebyLeiAndQuarter(lei) + + def deleteByLei(lei: String): Future[Int] = larRepo.deleteByLei(lei) + + private def getTableNameByYear(year: String): String = { + val configString = s"hmda.analytics.$year.larTableName$quarter" + config.getString(configString) + } + + } + //submission_history table remains same regardless of the year. There is a sign_date column and submission_id column which would show which year the filing was for val histTableName = config.getString("hmda.analytics.2018.historyTableName") - val tsTableName2019 = config.getString("hmda.analytics.2019.tsTableName") - val larTableName2019 = config.getString("hmda.analytics.2019.larTableName") - - //2020 TS - val tsTableName2020 = config.getString("hmda.analytics.2020.tsTableName") - val tsTableName2020Q1 = config.getString("hmda.analytics.2020.tsTableNameQ1") - val tsTableName2020Q2 = config.getString("hmda.analytics.2020.tsTableNameQ2") - val tsTableName2020Q3 = config.getString("hmda.analytics.2020.tsTableNameQ3") - - //2021 TS - val tsTableName2021 = config.getString("hmda.analytics.2021.tsTableName") - val tsTableName2021Q1 = config.getString("hmda.analytics.2021.tsTableNameQ1") - val tsTableName2021Q2 = config.getString("hmda.analytics.2021.tsTableNameQ2") - val tsTableName2021Q3 = config.getString("hmda.analytics.2021.tsTableNameQ3") - - //2020 LAR - val larTableName2020 = config.getString("hmda.analytics.2020.larTableName") - val larTableName2020Q1 = config.getString("hmda.analytics.2020.larTableNameQ1") - val larTableName2020Q2 = config.getString("hmda.analytics.2020.larTableNameQ2") - val larTableName2020Q3 = config.getString("hmda.analytics.2020.larTableNameQ3") - - //2021 LAR - val larTableName2021 = config.getString("hmda.analytics.2021.larTableName") - val larTableName2021Q1 = config.getString("hmda.analytics.2021.larTableNameQ1") - val larTableName2021Q2 = config.getString("hmda.analytics.2021.larTableNameQ2") - val larTableName2021Q3 = config.getString("hmda.analytics.2021.larTableNameQ3") - - - //2022 TS - val tsTableName2022 = config.getString("hmda.analytics.2022.tsTableName") - val tsTableName2022Q1 = config.getString("hmda.analytics.2022.tsTableNameQ1") - val tsTableName2022Q2 = config.getString("hmda.analytics.2022.tsTableNameQ2") - val tsTableName2022Q3 = config.getString("hmda.analytics.2022.tsTableNameQ3") - - //2022 LAR - val larTableName2022 = config.getString("hmda.analytics.2022.larTableName") - val larTableName2022Q1 = config.getString("hmda.analytics.2022.larTableNameQ1") - val larTableName2022Q2 = config.getString("hmda.analytics.2022.larTableNameQ2") - val larTableName2022Q3 = config.getString("hmda.analytics.2022.larTableNameQ3") - - - //2023 TS - val tsTableName2023 = config.getString("hmda.analytics.2023.tsTableName") - val tsTableName2023Q1 = config.getString("hmda.analytics.2023.tsTableNameQ1") - val tsTableName2023Q2 = config.getString("hmda.analytics.2023.tsTableNameQ2") - val tsTableName2023Q3 = config.getString("hmda.analytics.2023.tsTableNameQ3") - - //2023 LAR - val larTableName2023 = config.getString("hmda.analytics.2023.larTableName") - val larTableName2023Q1 = config.getString("hmda.analytics.2023.larTableNameQ1") - val larTableName2023Q2 = config.getString("hmda.analytics.2023.larTableNameQ2") - val larTableName2023Q3 = config.getString("hmda.analytics.2023.larTableNameQ3") - - //2024 TS - val tsTableName2024Q1 = config.getString("hmda.analytics.2024.tsTableNameQ1") - val tsTableName2024Q2 = config.getString("hmda.analytics.2024.tsTableNameQ2") - val tsTableName2024Q3 = config.getString("hmda.analytics.2023.tsTableNameQ3") - - //2024 LAR - val larTableName2024Q1 = config.getString("hmda.analytics.2024.larTableNameQ1") - val larTableName2024Q2 = config.getString("hmda.analytics.2024.larTableNameQ2") - val larTableName2024Q3 = config.getString("hmda.analytics.2024.larTableNameQ3") - - - - val transmittalSheetRepository2018 = new TransmittalSheetRepository(dbConfig, tsTableName2018) - val transmittalSheetRepository2019 = new TransmittalSheetRepository(dbConfig, tsTableName2019) - val transmittalSheetRepository2020 = new TransmittalSheetRepository(dbConfig, tsTableName2020) - val transmittalSheetRepository2021 = new TransmittalSheetRepository(dbConfig, tsTableName2021) - val transmittalSheetRepository2022 = new TransmittalSheetRepository(dbConfig, tsTableName2022) - val transmittalSheetRepository2023 = new TransmittalSheetRepository(dbConfig, tsTableName2023) - val transmittalSheetRepository2020Q1 = new TransmittalSheetRepository(dbConfig, tsTableName2020Q1) - val transmittalSheetRepository2020Q2 = new TransmittalSheetRepository(dbConfig, tsTableName2020Q2) - val transmittalSheetRepository2020Q3 = new TransmittalSheetRepository(dbConfig, tsTableName2020Q3) - val transmittalSheetRepository2021Q1 = new TransmittalSheetRepository(dbConfig, tsTableName2021Q1) - val transmittalSheetRepository2021Q2 = new TransmittalSheetRepository(dbConfig, tsTableName2021Q2) - val transmittalSheetRepository2021Q3 = new TransmittalSheetRepository(dbConfig, tsTableName2021Q3) - val transmittalSheetRepository2022Q1 = new TransmittalSheetRepository(dbConfig, tsTableName2022Q1) - val transmittalSheetRepository2022Q2 = new TransmittalSheetRepository(dbConfig, tsTableName2022Q2) - val transmittalSheetRepository2022Q3 = new TransmittalSheetRepository(dbConfig, tsTableName2022Q3) - val transmittalSheetRepository2023Q1 = new TransmittalSheetRepository(dbConfig, tsTableName2023Q1) - val transmittalSheetRepository2023Q2 = new TransmittalSheetRepository(dbConfig, tsTableName2023Q2) - val transmittalSheetRepository2023Q3 = new TransmittalSheetRepository(dbConfig, tsTableName2023Q3) - val transmittalSheetRepository2024Q1 = new TransmittalSheetRepository(dbConfig, tsTableName2024Q1) - val transmittalSheetRepository2024Q2 = new TransmittalSheetRepository(dbConfig, tsTableName2024Q2) - val transmittalSheetRepository2024Q3 = new TransmittalSheetRepository(dbConfig, tsTableName2024Q3) - val larRepository2018 = new LarRepository(dbConfig, larTableName2018) - val larRepository2019 = new LarRepository(dbConfig, larTableName2019) - val larRepository2020 = new LarRepository(dbConfig, larTableName2020) - val larRepository2021 = new LarRepository(dbConfig, larTableName2021) - val larRepository2022 = new LarRepository(dbConfig, larTableName2022) - val larRepository2023 = new LarRepository(dbConfig, larTableName2023) - - val larRepository2020Q1 = new LarRepository(dbConfig, larTableName2020Q1) - val larRepository2020Q2 = new LarRepository(dbConfig, larTableName2020Q2) - val larRepository2020Q3 = new LarRepository(dbConfig, larTableName2020Q3) - val larRepository2021Q1 = new LarRepository(dbConfig, larTableName2021Q1) - val larRepository2021Q2 = new LarRepository(dbConfig, larTableName2021Q2) - val larRepository2021Q3 = new LarRepository(dbConfig, larTableName2021Q3) - val larRepository2022Q1 = new LarRepository(dbConfig, larTableName2022Q1) - val larRepository2022Q2 = new LarRepository(dbConfig, larTableName2022Q2) - val larRepository2022Q3 = new LarRepository(dbConfig, larTableName2022Q3) - val larRepository2023Q1 = new LarRepository(dbConfig, larTableName2023Q1) - val larRepository2023Q2 = new LarRepository(dbConfig, larTableName2023Q2) - val larRepository2023Q3 = new LarRepository(dbConfig, larTableName2023Q3) - val larRepository2024Q1 = new LarRepository(dbConfig, larTableName2024Q1) - val larRepository2024Q2 = new LarRepository(dbConfig, larTableName2024Q2) - val larRepository2024Q3 = new LarRepository(dbConfig, larTableName2024Q3) + + + val submissionHistoryRepository = new SubmissionHistoryRepository(dbConfig, histTableName) val consumerSettings: ConsumerSettings[String, String] = @@ -405,27 +334,27 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo .mapAsync(1) { lar => for { delete <- submissionId.period match { - case Period(2018, None) => larRepository2018.deleteByLei(lar.larIdentifier.LEI) - case Period(2019, None) => larRepository2019.deleteByLei(lar.larIdentifier.LEI) - case Period(2020, Some("Q1")) => larRepository2020Q1.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2020, Some("Q2")) => larRepository2020Q2.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2020, Some("Q3")) => larRepository2020Q3.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2020, None) => larRepository2020.deleteByLei(lar.larIdentifier.LEI) - case Period(2021, None) => larRepository2021.deleteByLei(lar.larIdentifier.LEI) - case Period(2022, None) => larRepository2022.deleteByLei(lar.larIdentifier.LEI) - case Period(2023, None) => larRepository2023.deleteByLei(lar.larIdentifier.LEI) - case Period(2021, Some("Q1")) => larRepository2021Q1.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2021, Some("Q2")) => larRepository2021Q2.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2021, Some("Q3")) => larRepository2021Q3.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2022, Some("Q1")) => larRepository2022Q1.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2022, Some("Q2")) => larRepository2022Q2.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2022, Some("Q3")) => larRepository2022Q3.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2023, Some("Q1")) => larRepository2023Q1.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2023, Some("Q2")) => larRepository2023Q2.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2023, Some("Q3")) => larRepository2023Q3.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2024, Some("Q1")) => larRepository2024Q1.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2024, Some("Q2")) => larRepository2024Q2.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2024, Some("Q3")) => larRepository2024Q3.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2018, None) => YearlyLarRepositoryWrapper("2018").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2019, None) => YearlyLarRepositoryWrapper("2018").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2020, Some("Q1")) =>QuarterlyLarRepositoryWrapper("2020", "Q1").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2020, Some("Q2")) => QuarterlyLarRepositoryWrapper("2020", "Q2").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2020, Some("Q3")) => QuarterlyLarRepositoryWrapper("2020", "Q3").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2020, None) => YearlyLarRepositoryWrapper("2020").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2021, None) =>YearlyLarRepositoryWrapper("2021").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2022, None) =>YearlyLarRepositoryWrapper("2022").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2023, None) =>YearlyLarRepositoryWrapper("2023").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2021, Some("Q1")) => QuarterlyLarRepositoryWrapper("2021", "Q1").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2021, Some("Q2")) => QuarterlyLarRepositoryWrapper("2021", "Q2").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2021, Some("Q3")) => QuarterlyLarRepositoryWrapper("2021", "Q3").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2022, Some("Q1")) => QuarterlyLarRepositoryWrapper("2022", "Q1").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2022, Some("Q2")) => QuarterlyLarRepositoryWrapper("2022", "Q2").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2022, Some("Q3")) => QuarterlyLarRepositoryWrapper("2022", "Q3").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2023, Some("Q1")) => QuarterlyLarRepositoryWrapper("2023", "Q1").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2023, Some("Q2")) => QuarterlyLarRepositoryWrapper("2023", "Q2").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2023, Some("Q3")) => QuarterlyLarRepositoryWrapper("2023", "Q3").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2024, Some("Q1")) => QuarterlyLarRepositoryWrapper("2024", "Q1").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2024, Some("Q2")) => QuarterlyLarRepositoryWrapper("2024", "Q2").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2024, Some("Q3")) => QuarterlyLarRepositoryWrapper("2024", "Q3").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) case _ => { log.error(s"Unable to discern period from $submissionId to delete LAR rows.") throw new IllegalArgumentException(s"Unable to discern period from $submissionId to delete LAR rows.") @@ -450,85 +379,85 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo .mapAsync(1) { lar => for { insertorupdate <- submissionId.period match { - case Period(2018, None) => larRepository2018.insert(LarConverter2018(lar)) + case Period(2018, None) => YearlyLarRepositoryWrapper("2018").getLarRepository.insert(LarConverter2018(lar)) case Period(2019, None) => - larRepository2019.insert( + YearlyLarRepositoryWrapper("2018").getLarRepository.insert( LarConverter(lar, 2019) ) case Period(2020, Some("Q1")) => - larRepository2020Q1.insert( + QuarterlyLarRepositoryWrapper("2020", "Q1").getLarRepository.insert( LarConverter(lar = lar, 2020, isQuarterly = true) ) case Period(2020, Some("Q2")) => - larRepository2020Q2.insert( + QuarterlyLarRepositoryWrapper("2020", "Q2").getLarRepository.insert( LarConverter(lar = lar, 2020, isQuarterly = true) ) case Period(2020, Some("Q3")) => - larRepository2020Q3.insert( + QuarterlyLarRepositoryWrapper("2020", "Q3").getLarRepository.insert( LarConverter(lar = lar, 2020, isQuarterly = true) ) case Period(2020, None) => - larRepository2020.insert( + YearlyLarRepositoryWrapper("2020").getLarRepository.insert( LarConverter(lar = lar, 2020) ) case Period(2021, None) => - larRepository2021.insert( + YearlyLarRepositoryWrapper("2021").getLarRepository.insert( LarConverter(lar = lar, 2021) ) case Period(2022, None) => - larRepository2022.insert( + YearlyLarRepositoryWrapper("2022").getLarRepository.insert( LarConverter(lar = lar, 2022) ) case Period(2023, None) => - larRepository2023.insert( + YearlyLarRepositoryWrapper("2023").getLarRepository.insert( LarConverter(lar = lar, 2023) ) case Period(2021, Some("Q1")) => - larRepository2021Q1.insert( + QuarterlyLarRepositoryWrapper("2021", "Q1").getLarRepository.insert( LarConverter(lar = lar, 2021, isQuarterly = true) ) case Period(2021, Some("Q2")) => - larRepository2021Q2.insert( + QuarterlyLarRepositoryWrapper("2021", "Q2").getLarRepository.insert( LarConverter(lar = lar, 2021, isQuarterly = true) ) case Period(2021, Some("Q3")) => - larRepository2021Q3.insert( + QuarterlyLarRepositoryWrapper("2021", "Q3").getLarRepository.insert( LarConverter(lar = lar, 2021, isQuarterly = true) ) case Period(2022, Some("Q1")) => - larRepository2022Q1.insert( + QuarterlyLarRepositoryWrapper("2022", "Q1").getLarRepository.insert( LarConverter(lar = lar, 2022, isQuarterly = true) ) case Period(2022, Some("Q2")) => - larRepository2022Q2.insert( + QuarterlyLarRepositoryWrapper("2022", "Q2").getLarRepository.insert( LarConverter(lar = lar, 2022, isQuarterly = true) ) case Period(2022, Some("Q3")) => - larRepository2022Q3.insert( + QuarterlyLarRepositoryWrapper("2022", "Q3").getLarRepository.insert( LarConverter(lar = lar, 2022, isQuarterly = true) ) case Period(2023, Some("Q1")) => - larRepository2023Q1.insert( + QuarterlyLarRepositoryWrapper("2023", "Q1").getLarRepository.insert( LarConverter(lar = lar, 2023, isQuarterly = true) ) case Period(2023, Some("Q2")) => - larRepository2023Q2.insert( + QuarterlyLarRepositoryWrapper("2023", "Q2").getLarRepository.insert( LarConverter(lar = lar, 2023, isQuarterly = true) ) case Period(2023, Some("Q3")) => - larRepository2023Q3.insert( + QuarterlyLarRepositoryWrapper("2023", "Q3").getLarRepository.insert( LarConverter(lar = lar, 2023, isQuarterly = true) ) case Period(2024, Some("Q1")) => - larRepository2024Q1.insert( + QuarterlyLarRepositoryWrapper("2024", "Q1").getLarRepository.insert( LarConverter(lar = lar, 2024, isQuarterly = true) ) case Period(2024, Some("Q2")) => - larRepository2024Q2.insert( + QuarterlyLarRepositoryWrapper("2024", "Q2").getLarRepository.insert( LarConverter(lar = lar, 2024, isQuarterly = true) ) case Period(2024, Some("Q3")) => - larRepository2024Q3.insert( + QuarterlyLarRepositoryWrapper("2024", "Q3").getLarRepository.insert( LarConverter(lar = lar, 2024, isQuarterly = true) ) case _ => { From e3a3ea4c6c28ad7a65d438a6430c08ff704f7e97 Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Mon, 28 Oct 2024 13:07:07 -0400 Subject: [PATCH 06/19] Fix 2019 typo. --- .../src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index fbd90f9b1d..3ff7a226e4 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -335,7 +335,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo for { delete <- submissionId.period match { case Period(2018, None) => YearlyLarRepositoryWrapper("2018").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2019, None) => YearlyLarRepositoryWrapper("2018").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2019, None) => YearlyLarRepositoryWrapper("2019").getLarRepository.deleteByLei(lar.larIdentifier.LEI) case Period(2020, Some("Q1")) =>QuarterlyLarRepositoryWrapper("2020", "Q1").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) case Period(2020, Some("Q2")) => QuarterlyLarRepositoryWrapper("2020", "Q2").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) case Period(2020, Some("Q3")) => QuarterlyLarRepositoryWrapper("2020", "Q3").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) From 1f851b7ec650966567b8bf5de1b823c0c3e45dbb Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Mon, 28 Oct 2024 13:07:42 -0400 Subject: [PATCH 07/19] Fix 2019 typo. --- .../src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index 3ff7a226e4..da4970fa55 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -381,7 +381,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo insertorupdate <- submissionId.period match { case Period(2018, None) => YearlyLarRepositoryWrapper("2018").getLarRepository.insert(LarConverter2018(lar)) case Period(2019, None) => - YearlyLarRepositoryWrapper("2018").getLarRepository.insert( + YearlyLarRepositoryWrapper("2019").getLarRepository.insert( LarConverter(lar, 2019) ) case Period(2020, Some("Q1")) => From 85b99321eb8a0c8aad1daf9a47fb143af49d4c07 Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Tue, 29 Oct 2024 13:18:00 -0400 Subject: [PATCH 08/19] Remove unneeded params. --- .../src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index da4970fa55..58bc77d307 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -88,7 +88,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo def count(): Future[Int] = transmittalSheet.count() - private def getTableNameByYear(year: String): String = { + private def getTableNameByYear: String = { val configString = s"hmda.analytics.$year.tsTableName" config.getString(configString) } @@ -117,7 +117,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo def count(): Future[Int] = transmittalSheet.count() - private def getTableNameByYear(year: String): String = { + private def getTableNameByYear: String = { val configString = s"hmda.analytics.$year.tsTableName$quarter" config.getString(configString) } @@ -131,7 +131,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo def insert(le: LarEntity): Future[Int] = larRepo.insert(le) - private def getTableNameByYear(year: String): String = { + private def getTableNameByYear: String = { val configString = s"hmda.analytics.$year.larTableName" config.getString(configString) } @@ -153,7 +153,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo def deleteByLei(lei: String): Future[Int] = larRepo.deleteByLei(lei) - private def getTableNameByYear(year: String): String = { + private def getTableNameByYear: String = { val configString = s"hmda.analytics.$year.larTableName$quarter" config.getString(configString) } From c41c260ae21bffd3fd7625effc88259b07adcb43 Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Tue, 29 Oct 2024 13:24:26 -0400 Subject: [PATCH 09/19] Fix params. --- .../main/scala/hmda/analytics/HmdaAnalyticsApp.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index 58bc77d307..b8b5f9ac17 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -68,7 +68,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo */ case class YearlyTransmittalSheetRepositoryWrapper(year: String){ - val transmittalSheet = new TransmittalSheetRepository(dbConfig, getTableNameByYear(year)) + val transmittalSheet = new TransmittalSheetRepository(dbConfig, getTableNameByYear) def getTransmittalSheet = transmittalSheet @@ -88,7 +88,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo def count(): Future[Int] = transmittalSheet.count() - private def getTableNameByYear: String = { + private def getTableNameByYear = { val configString = s"hmda.analytics.$year.tsTableName" config.getString(configString) } @@ -96,7 +96,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo } case class QuarterlyTransmittalSheetRepositoryWrapper(year: String, quarter: String){ - val transmittalSheet = new TransmittalSheetRepository(dbConfig, getTableNameByYear(year)) + val transmittalSheet = new TransmittalSheetRepository(dbConfig, getTableNameByYear) def getTransmittalSheet = transmittalSheet @@ -125,7 +125,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo } case class YearlyLarRepositoryWrapper(year: String){ - val larRepo = new LarRepository(dbConfig, getTableNameByYear(year)) + val larRepo = new LarRepository(dbConfig, getTableNameByYear) def getLarRepository = larRepo @@ -139,7 +139,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo } case class QuarterlyLarRepositoryWrapper(year: String, quarter: String){ - val larRepo = new LarRepository(dbConfig, getTableNameByYear(year)) + val larRepo = new LarRepository(dbConfig, getTableNameByYear) def getLarRepository = larRepo From 64fd69cc617254a2faee6dcd275d48d293209d11 Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Wed, 30 Oct 2024 10:52:59 -0400 Subject: [PATCH 10/19] Remove unneeded function. --- .../src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index b8b5f9ac17..fdce77bf00 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -82,8 +82,6 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo def deleteByLei(lei: String): Future[Int] = transmittalSheet.deleteByLei(lei) - def deleteByLeiAndQuarter(lei: String): Future[Int] = transmittalSheet.deleteByLeiAndQuarter(lei) - def updateByLei(ts: TransmittalSheetEntity): Future[Int] = transmittalSheet.updateByLei(ts) def count(): Future[Int] = transmittalSheet.count() From fa870aa9b40eb6f92794ef8ad9902254264c9556 Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Wed, 30 Oct 2024 11:34:59 -0400 Subject: [PATCH 11/19] Remove old quarterly lei function. --- .../hmda/analytics/HmdaAnalyticsApp.scala | 34 ++++++++----------- .../query/TransmittalSheetComponent.scala | 3 -- .../query/TransmittalSheetComponentSpec.scala | 2 +- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index fdce77bf00..2013fd091c 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -109,8 +109,6 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo def deleteByLei(lei: String): Future[Int] = transmittalSheet.deleteByLei(lei) - def deleteByLeiAndQuarter(lei: String): Future[Int] = transmittalSheet.deleteByLeiAndQuarter(lei) - def updateByLei(ts: TransmittalSheetEntity): Future[Int] = transmittalSheet.updateByLei(ts) def count(): Future[Int] = transmittalSheet.count() @@ -146,8 +144,6 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo // for 2018 def insert(le: LarEntity2018): Future[Int] = larRepo.insert(le) - - def deletebyLeiAndQuarter(lei: String): Future[Int] = larRepo.deletebyLeiAndQuarter(lei) def deleteByLei(lei: String): Future[Int] = larRepo.deleteByLei(lei) @@ -334,25 +330,25 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo delete <- submissionId.period match { case Period(2018, None) => YearlyLarRepositoryWrapper("2018").getLarRepository.deleteByLei(lar.larIdentifier.LEI) case Period(2019, None) => YearlyLarRepositoryWrapper("2019").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2020, Some("Q1")) =>QuarterlyLarRepositoryWrapper("2020", "Q1").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2020, Some("Q2")) => QuarterlyLarRepositoryWrapper("2020", "Q2").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2020, Some("Q3")) => QuarterlyLarRepositoryWrapper("2020", "Q3").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2020, Some("Q1")) =>QuarterlyLarRepositoryWrapper("2020", "Q1").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2020, Some("Q2")) => QuarterlyLarRepositoryWrapper("2020", "Q2").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2020, Some("Q3")) => QuarterlyLarRepositoryWrapper("2020", "Q3").getLarRepository.deleteByLei(lar.larIdentifier.LEI) case Period(2020, None) => YearlyLarRepositoryWrapper("2020").getLarRepository.deleteByLei(lar.larIdentifier.LEI) case Period(2021, None) =>YearlyLarRepositoryWrapper("2021").getLarRepository.deleteByLei(lar.larIdentifier.LEI) case Period(2022, None) =>YearlyLarRepositoryWrapper("2022").getLarRepository.deleteByLei(lar.larIdentifier.LEI) case Period(2023, None) =>YearlyLarRepositoryWrapper("2023").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2021, Some("Q1")) => QuarterlyLarRepositoryWrapper("2021", "Q1").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2021, Some("Q2")) => QuarterlyLarRepositoryWrapper("2021", "Q2").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2021, Some("Q3")) => QuarterlyLarRepositoryWrapper("2021", "Q3").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2022, Some("Q1")) => QuarterlyLarRepositoryWrapper("2022", "Q1").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2022, Some("Q2")) => QuarterlyLarRepositoryWrapper("2022", "Q2").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2022, Some("Q3")) => QuarterlyLarRepositoryWrapper("2022", "Q3").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2023, Some("Q1")) => QuarterlyLarRepositoryWrapper("2023", "Q1").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2023, Some("Q2")) => QuarterlyLarRepositoryWrapper("2023", "Q2").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2023, Some("Q3")) => QuarterlyLarRepositoryWrapper("2023", "Q3").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2024, Some("Q1")) => QuarterlyLarRepositoryWrapper("2024", "Q1").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2024, Some("Q2")) => QuarterlyLarRepositoryWrapper("2024", "Q2").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) - case Period(2024, Some("Q3")) => QuarterlyLarRepositoryWrapper("2024", "Q3").getLarRepository.deletebyLeiAndQuarter(lar.larIdentifier.LEI) + case Period(2021, Some("Q1")) => QuarterlyLarRepositoryWrapper("2021", "Q1").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2021, Some("Q2")) => QuarterlyLarRepositoryWrapper("2021", "Q2").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2021, Some("Q3")) => QuarterlyLarRepositoryWrapper("2021", "Q3").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2022, Some("Q1")) => QuarterlyLarRepositoryWrapper("2022", "Q1").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2022, Some("Q2")) => QuarterlyLarRepositoryWrapper("2022", "Q2").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2022, Some("Q3")) => QuarterlyLarRepositoryWrapper("2022", "Q3").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2023, Some("Q1")) => QuarterlyLarRepositoryWrapper("2023", "Q1").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2023, Some("Q2")) => QuarterlyLarRepositoryWrapper("2023", "Q2").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2023, Some("Q3")) => QuarterlyLarRepositoryWrapper("2023", "Q3").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2024, Some("Q1")) => QuarterlyLarRepositoryWrapper("2024", "Q1").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2024, Some("Q2")) => QuarterlyLarRepositoryWrapper("2024", "Q2").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(2024, Some("Q3")) => QuarterlyLarRepositoryWrapper("2024", "Q3").getLarRepository.deleteByLei(lar.larIdentifier.LEI) case _ => { log.error(s"Unable to discern period from $submissionId to delete LAR rows.") throw new IllegalArgumentException(s"Unable to discern period from $submissionId to delete LAR rows.") diff --git a/hmda-analytics/src/main/scala/hmda/analytics/query/TransmittalSheetComponent.scala b/hmda-analytics/src/main/scala/hmda/analytics/query/TransmittalSheetComponent.scala index 51043b91ca..c280531a1a 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/query/TransmittalSheetComponent.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/query/TransmittalSheetComponent.scala @@ -87,9 +87,6 @@ trait TransmittalSheetComponent { def deleteByLei(lei: String): Future[Int] = db.run(table.filter(_.lei.toUpperCase === lei.toUpperCase).delete) - def deleteByLeiAndQuarter(lei: String): Future[Int] = - db.run(table.filter(x => x.lei.toUpperCase === lei.toUpperCase && x.isQuarterly === true).delete) - def updateByLei(ts: TransmittalSheetEntity): Future[Int] = db.run(table.insertOrUpdate(ts)) diff --git a/hmda-analytics/src/test/scala/hmda/analytics/query/TransmittalSheetComponentSpec.scala b/hmda-analytics/src/test/scala/hmda/analytics/query/TransmittalSheetComponentSpec.scala index 32930d7c00..a5e50cce3b 100644 --- a/hmda-analytics/src/test/scala/hmda/analytics/query/TransmittalSheetComponentSpec.scala +++ b/hmda-analytics/src/test/scala/hmda/analytics/query/TransmittalSheetComponentSpec.scala @@ -41,7 +41,7 @@ class TransmittalSheetComponentSpec extends TransmittalSheetAsyncSetup { _ = sheets.size mustBe 1 rowsUpdated <- transmittalSheetRepository.updateByLei(quarterlyTs.copy(name = quarterlyTs.name + "-updated")) _ = rowsUpdated mustBe 1 - rowsDeleted <- transmittalSheetRepository.deleteByLeiAndQuarter(quarterlyTs.lei) + rowsDeleted <- transmittalSheetRepository.deleteByLei(quarterlyTs.lei) } yield rowsDeleted mustBe 1 } } From 2f59941fa45032ef07a6de2c7ba617d2222a198f Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Fri, 1 Nov 2024 15:22:54 -0400 Subject: [PATCH 12/19] Simplify cases in deleteTsRow. --- .../hmda/analytics/HmdaAnalyticsApp.scala | 25 +++---------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index 2013fd091c..a760d95b8a 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -93,7 +93,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo } - case class QuarterlyTransmittalSheetRepositoryWrapper(year: String, quarter: String){ + case class QuarterlyTransmittalSheetRepositoryWrapper(year: String, quarter: Option[String]){ val transmittalSheet = new TransmittalSheetRepository(dbConfig, getTableNameByYear) def getTransmittalSheet = transmittalSheet @@ -208,27 +208,8 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo .mapAsync(1) { ts => for { delete <- submissionId.period match { - case Period(2018, None) => YearlyTransmittalSheetRepositoryWrapper("2018").deleteByLei(ts.lei) - case Period(2019, None) => YearlyTransmittalSheetRepositoryWrapper("2019").deleteByLei(ts.lei) - case Period(2020, Some("Q1")) => QuarterlyTransmittalSheetRepositoryWrapper("2020", "Q1").deleteByLei(ts.lei) - case Period(2020, Some("Q2")) => QuarterlyTransmittalSheetRepositoryWrapper("2020", "Q2").deleteByLei(ts.lei) - case Period(2020, Some("Q3")) => QuarterlyTransmittalSheetRepositoryWrapper("2020", "Q3").deleteByLei(ts.lei) - case Period(2020, None) => YearlyTransmittalSheetRepositoryWrapper("2020").deleteByLei(ts.lei) - case Period(2021, None) => YearlyTransmittalSheetRepositoryWrapper("2021").deleteByLei(ts.lei) - case Period(2022, None) => YearlyTransmittalSheetRepositoryWrapper("2022").deleteByLei(ts.lei) - case Period(2023, None) => YearlyTransmittalSheetRepositoryWrapper("2023").deleteByLei(ts.lei) - case Period(2021, Some("Q1")) => QuarterlyTransmittalSheetRepositoryWrapper("2021", "Q1").deleteByLei(ts.lei) - case Period(2021, Some("Q2")) => QuarterlyTransmittalSheetRepositoryWrapper("2021", "Q2").deleteByLei(ts.lei) - case Period(2021, Some("Q3")) => QuarterlyTransmittalSheetRepositoryWrapper("2021", "Q3").deleteByLei(ts.lei) - case Period(2022, Some("Q1")) => QuarterlyTransmittalSheetRepositoryWrapper("2022", "Q1").deleteByLei(ts.lei) - case Period(2022, Some("Q2")) => QuarterlyTransmittalSheetRepositoryWrapper("2022", "Q2").deleteByLei(ts.lei) - case Period(2022, Some("Q3")) => QuarterlyTransmittalSheetRepositoryWrapper("2022", "Q3").deleteByLei(ts.lei) - case Period(2023, Some("Q1")) => QuarterlyTransmittalSheetRepositoryWrapper("2023", "Q1").deleteByLei(ts.lei) - case Period(2023, Some("Q2")) => QuarterlyTransmittalSheetRepositoryWrapper("2023", "Q2").deleteByLei(ts.lei) - case Period(2023, Some("Q3")) => QuarterlyTransmittalSheetRepositoryWrapper("2023", "Q3").deleteByLei(ts.lei) - case Period(2024, Some("Q1")) => QuarterlyTransmittalSheetRepositoryWrapper("2024", "Q1").deleteByLei(ts.lei) - case Period(2024, Some("Q2")) => QuarterlyTransmittalSheetRepositoryWrapper("2024", "Q2").deleteByLei(ts.lei) - case Period(2024, Some("Q3")) => QuarterlyTransmittalSheetRepositoryWrapper("2024", "Q3").deleteByLei(ts.lei) + case Period(submissionId.period.year, None) => YearlyTransmittalSheetRepositoryWrapper(submissionId.period.year.toString).deleteByLei(ts.lei) + case Period(submissionId.period.year, Some(submissionId.period.quarter)) => QuarterlyTransmittalSheetRepositoryWrapper(submissionId.period.year.toString, submissionId.period.quarter).deleteByLei(ts.lei) case _ => { log.error(s"Unable to discern period from $submissionId to delete TS rows.") throw new IllegalArgumentException(s"Unable to discern period from $submissionId to delete TS rows.") From eba19f0b82b2e7747536bc589fd32cbaf6659acb Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Fri, 1 Nov 2024 15:34:31 -0400 Subject: [PATCH 13/19] Simplify cases in insertTsRow. --- .../hmda/analytics/HmdaAnalyticsApp.scala | 23 ++----------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index a760d95b8a..230b59c18e 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -253,27 +253,8 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo .map(ts => TransmittalSheetConverter(ts, submissionIdOption)) .mapAsync(1) { ts => val (repo, enforceQuarterly) = submissionId.period match { - case Period(2018, None) => (YearlyTransmittalSheetRepositoryWrapper("2018").getTransmittalSheet, false) - case Period(2019, None) => (YearlyTransmittalSheetRepositoryWrapper("2019").getTransmittalSheet, false) - case Period(2020, Some("Q1")) => (QuarterlyTransmittalSheetRepositoryWrapper("2020", "Q1").getTransmittalSheet, true) - case Period(2020, Some("Q2")) => (QuarterlyTransmittalSheetRepositoryWrapper("2020", "Q2").getTransmittalSheet, true) - case Period(2020, Some("Q3")) => (QuarterlyTransmittalSheetRepositoryWrapper("2020", "Q3").getTransmittalSheet, true) - case Period(2020, None) => (YearlyTransmittalSheetRepositoryWrapper("2020").getTransmittalSheet, false) - case Period(2021, None) => (YearlyTransmittalSheetRepositoryWrapper("2021").getTransmittalSheet, false) - case Period(2022, None) => (YearlyTransmittalSheetRepositoryWrapper("2022").getTransmittalSheet, false) - case Period(2023, None) => (YearlyTransmittalSheetRepositoryWrapper("2023").getTransmittalSheet, false) - case Period(2021, Some("Q1")) => (QuarterlyTransmittalSheetRepositoryWrapper("2021", "Q1").getTransmittalSheet, true) - case Period(2021, Some("Q2")) => (QuarterlyTransmittalSheetRepositoryWrapper("2021", "Q2").getTransmittalSheet, true) - case Period(2021, Some("Q3")) => (QuarterlyTransmittalSheetRepositoryWrapper("2021", "Q3").getTransmittalSheet, true) - case Period(2022, Some("Q1")) => (QuarterlyTransmittalSheetRepositoryWrapper("2022", "Q1").getTransmittalSheet, true) - case Period(2022, Some("Q2")) => (QuarterlyTransmittalSheetRepositoryWrapper("2022", "Q2").getTransmittalSheet, true) - case Period(2022, Some("Q3")) => (QuarterlyTransmittalSheetRepositoryWrapper("2022", "Q3").getTransmittalSheet, true) - case Period(2023, Some("Q1")) => (QuarterlyTransmittalSheetRepositoryWrapper("2023", "Q1").getTransmittalSheet, true) - case Period(2023, Some("Q2")) => (QuarterlyTransmittalSheetRepositoryWrapper("2023", "Q2").getTransmittalSheet, true) - case Period(2023, Some("Q3")) => (QuarterlyTransmittalSheetRepositoryWrapper("2023", "Q3").getTransmittalSheet, true) - case Period(2024, Some("Q1")) => (QuarterlyTransmittalSheetRepositoryWrapper("2024", "Q1").getTransmittalSheet, true) - case Period(2024, Some("Q2")) => (QuarterlyTransmittalSheetRepositoryWrapper("2024", "Q2").getTransmittalSheet, true) - case Period(2024, Some("Q3")) => (QuarterlyTransmittalSheetRepositoryWrapper("2024", "Q3").getTransmittalSheet, true) + case Period(submissionId.period.year, None) => (YearlyTransmittalSheetRepositoryWrapper(submissionId.period.year.toString).getTransmittalSheet, false) + case Period(submissionId.period.year, Some(submissionId.period.quarter)) => (QuarterlyTransmittalSheetRepositoryWrapper(submissionId.period.year.toString, submissionId.period.quarter).getTransmittalSheet, true) case _ =>{ log.error(s"Unable to discern period from $submissionId to insert TS rows.") throw new IllegalArgumentException(s"Unable to discern period from $submissionId to insert TS rows.") From edb6c1eb9a51e28c1e8cbed78c84f880406dc930 Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Fri, 1 Nov 2024 15:38:17 -0400 Subject: [PATCH 14/19] Simplify cases in deleteLarRow. --- .../hmda/analytics/HmdaAnalyticsApp.scala | 25 +++---------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index 230b59c18e..67fa07a8f3 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -134,7 +134,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo } - case class QuarterlyLarRepositoryWrapper(year: String, quarter: String){ + case class QuarterlyLarRepositoryWrapper(year: String, quarter: Option[String]){ val larRepo = new LarRepository(dbConfig, getTableNameByYear) def getLarRepository = larRepo @@ -290,27 +290,8 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo .mapAsync(1) { lar => for { delete <- submissionId.period match { - case Period(2018, None) => YearlyLarRepositoryWrapper("2018").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2019, None) => YearlyLarRepositoryWrapper("2019").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2020, Some("Q1")) =>QuarterlyLarRepositoryWrapper("2020", "Q1").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2020, Some("Q2")) => QuarterlyLarRepositoryWrapper("2020", "Q2").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2020, Some("Q3")) => QuarterlyLarRepositoryWrapper("2020", "Q3").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2020, None) => YearlyLarRepositoryWrapper("2020").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2021, None) =>YearlyLarRepositoryWrapper("2021").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2022, None) =>YearlyLarRepositoryWrapper("2022").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2023, None) =>YearlyLarRepositoryWrapper("2023").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2021, Some("Q1")) => QuarterlyLarRepositoryWrapper("2021", "Q1").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2021, Some("Q2")) => QuarterlyLarRepositoryWrapper("2021", "Q2").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2021, Some("Q3")) => QuarterlyLarRepositoryWrapper("2021", "Q3").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2022, Some("Q1")) => QuarterlyLarRepositoryWrapper("2022", "Q1").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2022, Some("Q2")) => QuarterlyLarRepositoryWrapper("2022", "Q2").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2022, Some("Q3")) => QuarterlyLarRepositoryWrapper("2022", "Q3").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2023, Some("Q1")) => QuarterlyLarRepositoryWrapper("2023", "Q1").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2023, Some("Q2")) => QuarterlyLarRepositoryWrapper("2023", "Q2").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2023, Some("Q3")) => QuarterlyLarRepositoryWrapper("2023", "Q3").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2024, Some("Q1")) => QuarterlyLarRepositoryWrapper("2024", "Q1").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2024, Some("Q2")) => QuarterlyLarRepositoryWrapper("2024", "Q2").getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(2024, Some("Q3")) => QuarterlyLarRepositoryWrapper("2024", "Q3").getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(submissionId.period.year, None) => YearlyLarRepositoryWrapper(submissionId.period.year.toString).getLarRepository.deleteByLei(lar.larIdentifier.LEI) + case Period(submissionId.period.year, Some(submissionId.period.quarter)) => QuarterlyLarRepositoryWrapper(submissionId.period.year.toString, submissionId.period.quarter).deleteByLei(lar.larIdentifier.LEI) case _ => { log.error(s"Unable to discern period from $submissionId to delete LAR rows.") throw new IllegalArgumentException(s"Unable to discern period from $submissionId to delete LAR rows.") From 4dd0be32a8c4e1dfe73265aec6a9817349743738 Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Fri, 1 Nov 2024 15:44:08 -0400 Subject: [PATCH 15/19] Simplify cases in insertLarRows.. --- .../hmda/analytics/HmdaAnalyticsApp.scala | 85 ++----------------- 1 file changed, 6 insertions(+), 79 deletions(-) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index 67fa07a8f3..cb22bb6816 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -317,85 +317,12 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo for { insertorupdate <- submissionId.period match { case Period(2018, None) => YearlyLarRepositoryWrapper("2018").getLarRepository.insert(LarConverter2018(lar)) - case Period(2019, None) => - YearlyLarRepositoryWrapper("2019").getLarRepository.insert( - LarConverter(lar, 2019) - ) - case Period(2020, Some("Q1")) => - QuarterlyLarRepositoryWrapper("2020", "Q1").getLarRepository.insert( - LarConverter(lar = lar, 2020, isQuarterly = true) - ) - case Period(2020, Some("Q2")) => - QuarterlyLarRepositoryWrapper("2020", "Q2").getLarRepository.insert( - LarConverter(lar = lar, 2020, isQuarterly = true) - ) - case Period(2020, Some("Q3")) => - QuarterlyLarRepositoryWrapper("2020", "Q3").getLarRepository.insert( - LarConverter(lar = lar, 2020, isQuarterly = true) - ) - case Period(2020, None) => - YearlyLarRepositoryWrapper("2020").getLarRepository.insert( - LarConverter(lar = lar, 2020) - ) - case Period(2021, None) => - YearlyLarRepositoryWrapper("2021").getLarRepository.insert( - LarConverter(lar = lar, 2021) - ) - case Period(2022, None) => - YearlyLarRepositoryWrapper("2022").getLarRepository.insert( - LarConverter(lar = lar, 2022) - ) - case Period(2023, None) => - YearlyLarRepositoryWrapper("2023").getLarRepository.insert( - LarConverter(lar = lar, 2023) - ) - case Period(2021, Some("Q1")) => - QuarterlyLarRepositoryWrapper("2021", "Q1").getLarRepository.insert( - LarConverter(lar = lar, 2021, isQuarterly = true) - ) - case Period(2021, Some("Q2")) => - QuarterlyLarRepositoryWrapper("2021", "Q2").getLarRepository.insert( - LarConverter(lar = lar, 2021, isQuarterly = true) - ) - case Period(2021, Some("Q3")) => - QuarterlyLarRepositoryWrapper("2021", "Q3").getLarRepository.insert( - LarConverter(lar = lar, 2021, isQuarterly = true) - ) - case Period(2022, Some("Q1")) => - QuarterlyLarRepositoryWrapper("2022", "Q1").getLarRepository.insert( - LarConverter(lar = lar, 2022, isQuarterly = true) - ) - case Period(2022, Some("Q2")) => - QuarterlyLarRepositoryWrapper("2022", "Q2").getLarRepository.insert( - LarConverter(lar = lar, 2022, isQuarterly = true) - ) - case Period(2022, Some("Q3")) => - QuarterlyLarRepositoryWrapper("2022", "Q3").getLarRepository.insert( - LarConverter(lar = lar, 2022, isQuarterly = true) - ) - case Period(2023, Some("Q1")) => - QuarterlyLarRepositoryWrapper("2023", "Q1").getLarRepository.insert( - LarConverter(lar = lar, 2023, isQuarterly = true) - ) - case Period(2023, Some("Q2")) => - QuarterlyLarRepositoryWrapper("2023", "Q2").getLarRepository.insert( - LarConverter(lar = lar, 2023, isQuarterly = true) - ) - case Period(2023, Some("Q3")) => - QuarterlyLarRepositoryWrapper("2023", "Q3").getLarRepository.insert( - LarConverter(lar = lar, 2023, isQuarterly = true) - ) - case Period(2024, Some("Q1")) => - QuarterlyLarRepositoryWrapper("2024", "Q1").getLarRepository.insert( - LarConverter(lar = lar, 2024, isQuarterly = true) - ) - case Period(2024, Some("Q2")) => - QuarterlyLarRepositoryWrapper("2024", "Q2").getLarRepository.insert( - LarConverter(lar = lar, 2024, isQuarterly = true) - ) - case Period(2024, Some("Q3")) => - QuarterlyLarRepositoryWrapper("2024", "Q3").getLarRepository.insert( - LarConverter(lar = lar, 2024, isQuarterly = true) + case Period(submissionId.period.year, None) => YearlyLarRepositoryWrapper(submissionId.period.year.toString).getLarRepository.insert( + LarConverter(lar, submissionId.period.year) + ) + case Period(submissionId.period.year, Some(submissionId.period.quarter)) => + QuarterlyLarRepositoryWrapper(submissionId.period.year.toString, submissionId.period.quarter).getLarRepository.insert( + LarConverter(lar = lar, submissionId.period.year, isQuarterly = true) ) case _ => { log.error(s"Unable to discern period from $submissionId to insert LAR rows.") From 31eca7a3a480d5046a9ebcb6255b8a9cbe9d1ee6 Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Fri, 1 Nov 2024 15:53:45 -0400 Subject: [PATCH 16/19] Fix compilation error. --- .../src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala index cb22bb6816..a1b6623a5c 100644 --- a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala +++ b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala @@ -209,7 +209,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo for { delete <- submissionId.period match { case Period(submissionId.period.year, None) => YearlyTransmittalSheetRepositoryWrapper(submissionId.period.year.toString).deleteByLei(ts.lei) - case Period(submissionId.period.year, Some(submissionId.period.quarter)) => QuarterlyTransmittalSheetRepositoryWrapper(submissionId.period.year.toString, submissionId.period.quarter).deleteByLei(ts.lei) + case Period(submissionId.period.year, submissionId.period.quarter) => QuarterlyTransmittalSheetRepositoryWrapper(submissionId.period.year.toString, submissionId.period.quarter).deleteByLei(ts.lei) case _ => { log.error(s"Unable to discern period from $submissionId to delete TS rows.") throw new IllegalArgumentException(s"Unable to discern period from $submissionId to delete TS rows.") @@ -254,7 +254,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo .mapAsync(1) { ts => val (repo, enforceQuarterly) = submissionId.period match { case Period(submissionId.period.year, None) => (YearlyTransmittalSheetRepositoryWrapper(submissionId.period.year.toString).getTransmittalSheet, false) - case Period(submissionId.period.year, Some(submissionId.period.quarter)) => (QuarterlyTransmittalSheetRepositoryWrapper(submissionId.period.year.toString, submissionId.period.quarter).getTransmittalSheet, true) + case Period(submissionId.period.year, submissionId.period.quarter) => (QuarterlyTransmittalSheetRepositoryWrapper(submissionId.period.year.toString, submissionId.period.quarter).getTransmittalSheet, true) case _ =>{ log.error(s"Unable to discern period from $submissionId to insert TS rows.") throw new IllegalArgumentException(s"Unable to discern period from $submissionId to insert TS rows.") @@ -291,7 +291,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo for { delete <- submissionId.period match { case Period(submissionId.period.year, None) => YearlyLarRepositoryWrapper(submissionId.period.year.toString).getLarRepository.deleteByLei(lar.larIdentifier.LEI) - case Period(submissionId.period.year, Some(submissionId.period.quarter)) => QuarterlyLarRepositoryWrapper(submissionId.period.year.toString, submissionId.period.quarter).deleteByLei(lar.larIdentifier.LEI) + case Period(submissionId.period.year, submissionId.period.quarter) => QuarterlyLarRepositoryWrapper(submissionId.period.year.toString, submissionId.period.quarter).deleteByLei(lar.larIdentifier.LEI) case _ => { log.error(s"Unable to discern period from $submissionId to delete LAR rows.") throw new IllegalArgumentException(s"Unable to discern period from $submissionId to delete LAR rows.") @@ -320,7 +320,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo case Period(submissionId.period.year, None) => YearlyLarRepositoryWrapper(submissionId.period.year.toString).getLarRepository.insert( LarConverter(lar, submissionId.period.year) ) - case Period(submissionId.period.year, Some(submissionId.period.quarter)) => + case Period(submissionId.period.year, submissionId.period.quarter) => QuarterlyLarRepositoryWrapper(submissionId.period.year.toString, submissionId.period.quarter).getLarRepository.insert( LarConverter(lar = lar, submissionId.period.year, isQuarterly = true) ) From db184bb935ef8c22fd4dde70b94adf8208510f03 Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Fri, 29 Nov 2024 19:24:33 -0500 Subject: [PATCH 17/19] Add 2025 Q1 for devtesting. --- hmda-analytics/src/main/resources/application.conf | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/hmda-analytics/src/main/resources/application.conf b/hmda-analytics/src/main/resources/application.conf index 3cc1f5c774..09d99a32d5 100644 --- a/hmda-analytics/src/main/resources/application.conf +++ b/hmda-analytics/src/main/resources/application.conf @@ -62,8 +62,18 @@ hmda { larDeletion = ${?HMDA-LAR-DELETE} historyInsertion = true historyInsertion = ${?HMDA-HISTORY-INSERT} - yearsAvailable = "2018,2019,2020,2021,2022,2023,2024" + yearsAvailable = "2018,2019,2020,2021,2022,2023,2024,2025" yearsAvailable = ${?ANALYTICS_YEARS_AVAILABLE} + 2025 { + tsTableName = "transmittalsheet2025" + tsTableName = ${?TS_TABLE_NAME_2025} + tsTableNameQ1 = "ts2025_q1" + tsTableNameQ1 = ${?TS_TABLE_NAME_2025_Q1} + larTableName = "loanapplicationregister2025" + larTableName = ${?LAR_TABLE_NAME_2025} + larTableNameQ1 = "lar2025_q1" + larTableNameQ1 = ${?LAR_TABLE_NAME_2025_Q1} + } 2024 { tsTableName = "transmittalsheet2024" tsTableName = ${?TS_TABLE_NAME_2024} From efa74b0983a232aef654555fb50919c8c5bf3971 Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Fri, 29 Nov 2024 19:26:33 -0500 Subject: [PATCH 18/19] Revert 2025. --- hmda-analytics/src/main/resources/application.conf | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/hmda-analytics/src/main/resources/application.conf b/hmda-analytics/src/main/resources/application.conf index 09d99a32d5..cc7f0c904d 100644 --- a/hmda-analytics/src/main/resources/application.conf +++ b/hmda-analytics/src/main/resources/application.conf @@ -64,16 +64,6 @@ hmda { historyInsertion = ${?HMDA-HISTORY-INSERT} yearsAvailable = "2018,2019,2020,2021,2022,2023,2024,2025" yearsAvailable = ${?ANALYTICS_YEARS_AVAILABLE} - 2025 { - tsTableName = "transmittalsheet2025" - tsTableName = ${?TS_TABLE_NAME_2025} - tsTableNameQ1 = "ts2025_q1" - tsTableNameQ1 = ${?TS_TABLE_NAME_2025_Q1} - larTableName = "loanapplicationregister2025" - larTableName = ${?LAR_TABLE_NAME_2025} - larTableNameQ1 = "lar2025_q1" - larTableNameQ1 = ${?LAR_TABLE_NAME_2025_Q1} - } 2024 { tsTableName = "transmittalsheet2024" tsTableName = ${?TS_TABLE_NAME_2024} From de39641911c31e92e7d1379f18a3df8a228efd7f Mon Sep 17 00:00:00 2001 From: jaredb96 Date: Mon, 2 Dec 2024 08:53:01 -0500 Subject: [PATCH 19/19] Revert 2025. --- hmda-analytics/src/main/resources/application.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hmda-analytics/src/main/resources/application.conf b/hmda-analytics/src/main/resources/application.conf index cc7f0c904d..3cc1f5c774 100644 --- a/hmda-analytics/src/main/resources/application.conf +++ b/hmda-analytics/src/main/resources/application.conf @@ -62,7 +62,7 @@ hmda { larDeletion = ${?HMDA-LAR-DELETE} historyInsertion = true historyInsertion = ${?HMDA-HISTORY-INSERT} - yearsAvailable = "2018,2019,2020,2021,2022,2023,2024,2025" + yearsAvailable = "2018,2019,2020,2021,2022,2023,2024" yearsAvailable = ${?ANALYTICS_YEARS_AVAILABLE} 2024 { tsTableName = "transmittalsheet2024"