Jim/1244 org filters (#3274)

* Rearchitected ReportStream's Filters: Created Organization-level filters, new routingFilter, new processingMode filter, and defaults for all four filter types.
CDCgov · Dec 13, 2021 · ad23175 · ad23175
1 parent 583e5ea
commit ad23175
Show file tree

Hide file tree

Showing 15 changed files with 788 additions and 273 deletions.
diff --git a/.environment/gitleaks/gitleaks-config.toml b/.environment/gitleaks/gitleaks-config.toml
@@ -120,6 +120,7 @@ title = "PRIME ReportStream Gitleaks Configuration"
     ]
     [rules.allowlist]
         regexes = [
+            'api/lookuptables/list',
             ' \* \(The older version of this API is "/api/reports"\)',
             ' \* since this auth (has|uses) a ',
             ' by option\(\"',

diff --git a/prime-router/settings/organizations.yml b/prime-router/settings/organizations.yml
@@ -1770,6 +1770,9 @@
 - name: ignore
   description: FOR TESTING ONLY
   jurisdiction: FEDERAL
+  filters:
+    - topic: covid-19
+      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)" ]
   senders:
     - name: ignore-strac      # Use this to test sending strac data
       organizationName: ignore
@@ -1807,7 +1810,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: inactive
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, CSV)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, CSV)" ]
       timing:
         operation: MERGE
         numberPerDay: 1440 # Every minute
@@ -1827,7 +1830,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: inactive
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, HL7)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, HL7)" ]
       timing:
         operation: MERGE
         numberPerDay: 1440 # Every minute
@@ -1847,7 +1850,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: inactive
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, HL7_BATCH)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, HL7_BATCH)" ]
       timing:
         operation: MERGE
         numberPerDay: 1440 # Every minute
@@ -1866,7 +1869,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: inactive
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, REDOX)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, REDOX)" ]
       timing:
         operation: MERGE
         numberPerDay: 1440 # Every minute
@@ -1889,7 +1892,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: inactive
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, AS2)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, AS2)" ]
       timing:
         operation: MERGE
         numberPerDay: 1440 # Every minute
@@ -1901,7 +1904,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: inactive
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, HL7_NULL)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, HL7_NULL)" ]
       timing:
         operation: MERGE
         numberPerDay: 1440 # Every minute
@@ -1914,7 +1917,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: inactive
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, BLOBSTORE)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, BLOBSTORE)" ]
       timing:
         operation: MERGE
         numberPerDay: 1440 # Every minute
@@ -1931,7 +1934,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: inactive
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, SFTP_FAIL)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, SFTP_FAIL)" ]
       timing:
         operation: MERGE
         numberPerDay: 1440 # Every minute
@@ -1949,7 +1952,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: inactive
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, QUALITY_PASS, removed)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, QUALITY_PASS, removed)" ]
       qualityFilter:
         - hasValidDataFor(message_id,ordering_facility_county,ordering_facility_state)
         - hasAtLeastOneOf(message_id,blankField)
@@ -1962,7 +1965,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: inactive
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, QUALITY_REVERSED, kept)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, QUALITY_REVERSED, kept)" ]
       qualityFilter:
         - hasValidDataFor(message_id,ordering_facility_county,ordering_facility_state)
         - hasAtLeastOneOf(message_id,blankField)
@@ -1977,7 +1980,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: inactive
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, QUALITY_ALL)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, QUALITY_ALL)" ]
       qualityFilter: [ "allowAll()" ]
       translation:
         type: CUSTOM
@@ -1987,7 +1990,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: inactive
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, QUALITY_FAIL)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, QUALITY_FAIL)" ]
       qualityFilter: [ "hasValidDataFor(blankField)", "hasAtLeastOneOf(message_id,blankField)" ]
       translation:
         type: CUSTOM
@@ -1997,7 +2000,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: inactive
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, OTC_PROCTORED)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, OTC_PROCTORED)" ]
       qualityFilter: [ "matches(test_authorized_for_otc, Y)","matches(test_authorized_for_home, Y)","matches(test_authorized_for_unproctored, Y)" ]
       translation:
         type: "CUSTOM"
@@ -2008,7 +2011,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: inactive
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, OTC_PROCTORED)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, OTC_PROCTORED)" ]
       qualityFilter: [ "matches(test_authorized_for_otc, N)","matches(test_authorized_for_home, Y)","matches(test_authorized_for_unproctored, Y)" ]
       translation:
         type: "CUSTOM"
@@ -2019,7 +2022,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: inactive
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, OTC_PROCTORED)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, OTC_PROCTORED)" ]
       qualityFilter: [ "matches(test_authorized_for_otc, N)","matches(test_authorized_for_home, UNK)","matches(test_authorized_for_unproctored, UNK)" ]
       translation:
         type: "CUSTOM"
@@ -2031,7 +2034,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: active
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, EVERY_5_MINS)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, EVERY_5_MINS)" ]
       timing:
         operation: MERGE
         numberPerDay: 288 # Every 5 minutes
@@ -2050,7 +2053,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: active
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, EVERY_15_MINS)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, EVERY_15_MINS)" ]
       timing:
         operation: MERGE
         numberPerDay: 96 # Every 15 minutes
@@ -2069,7 +2072,7 @@
       organizationName: ignore
       topic: covid-19
       customerStatus: active
-      jurisdictionalFilter: [ "matches(ordering_facility_state, IG)", "matches(ordering_facility_county, EVERY_60_MINS)" ]
+      jurisdictionalFilter: [ "matches(ordering_facility_county, EVERY_60_MINS)" ]
       timing:
         operation: MERGE
         numberPerDay: 24 # Every 60 minutes

diff --git a/prime-router/src/main/kotlin/Metadata.kt b/prime-router/src/main/kotlin/Metadata.kt
@@ -45,7 +45,7 @@ class Metadata : Logging {
         HashMapper(),
         NullMapper()
     )
-    private var jurisdictionalFilters = listOf(
+    private var reportStreamFilterDefinitions = listOf(
         FilterByCounty(),
         Matches(),
         DoesNotMatch(),
@@ -252,11 +252,11 @@ class Metadata : Logging {
     }
 
     /*
-    * JurisdictionalFilters
+    * ReportStreamFilterDefinitions
     */
 
-    fun findJurisdictionalFilter(name: String): JurisdictionalFilter? {
-        return jurisdictionalFilters.find { it.name.equals(name, ignoreCase = true) }
+    fun findReportStreamFilterDefinitions(name: String): ReportStreamFilterDefinition? {
+        return reportStreamFilterDefinitions.find { it.name.equals(name, ignoreCase = true) }
     }
 
     /*

diff --git a/prime-router/src/main/kotlin/Organization.kt b/prime-router/src/main/kotlin/Organization.kt
@@ -9,6 +9,7 @@ open class Organization(
     val jurisdiction: Jurisdiction,
     val stateCode: String?,
     val countyName: String?,
+    val filters: List<ReportStreamFilters>? = emptyList(), // one ReportStreamFilters obj per topic. 
 ) {
     constructor(org: Organization) : this(org.name, org.description, org.jurisdiction, org.stateCode, org.countyName)
 
@@ -53,9 +54,13 @@ class DeepOrganization(
     jurisdiction: Jurisdiction,
     stateCode: String? = null,
     countyName: String? = null,
+    filters: List<ReportStreamFilters>? = emptyList(),
     val senders: List<Sender> = emptyList(),
     val receivers: List<Receiver> = emptyList(),
-) : Organization(name, description, jurisdiction, stateCode, countyName) {
+) : Organization(name, description, jurisdiction, stateCode, countyName, filters) {
     constructor(org: Organization, senders: List<Sender>, receivers: List<Receiver>) :
-        this(org.name, org.description, org.jurisdiction, org.stateCode, org.countyName, senders, receivers)
+        this(
+            org.name, org.description, org.jurisdiction, org.stateCode, org.countyName, org.filters,
+            senders, receivers
+        )
 }
diff --git a/prime-router/src/main/kotlin/Receiver.kt b/prime-router/src/main/kotlin/Receiver.kt
@@ -15,8 +15,12 @@ import java.time.ZoneId
  * @param topic defines the set of schemas that can translate to each other
  * @param customerStatus defines if the receiver is fully onboarded
  * @param translation configuration to translate
- * @param jurisdictionalFilter defines the set of elements and regexes that filter the data for this receiver
- * @param qualityFilter defines the set of elements and regexes that do qualiyty filtering on the data for this receiver
+ * @param jurisdictionalFilter defines the geographic region filters for this receiver
+ * @param qualityFilter defines the filters that remove data, based on quality criteria
+ * @param routingFilter The original use case was for filters that remove data the
+ * receiver does not want, based on who sent it.  However, its available for any general purpose use.
+ * @param processingModeFilter defines the filters that is normally set to remove test and debug data.
+ * @param reverseTheQualityFilter If this is true, then do the NOT of 'qualityFilter'.  Like a 'grep -v'
  * @param deidentify transform
  * @param timing defines how to delay reports to the org. If null, then send immediately
  * @param description of the receiver
@@ -29,9 +33,10 @@ open class Receiver(
     val topic: String,
     val customerStatus: CustomerStatus = CustomerStatus.INACTIVE,
     val translation: TranslatorConfiguration,
-    val jurisdictionalFilter: List<String> = emptyList(),
-    val qualityFilter: List<String> = emptyList(),
-    // If this is true, then do the NOT of 'qualityFilter'.  Like a 'grep -v'
+    val jurisdictionalFilter: ReportStreamFilter = emptyList(),
+    val qualityFilter: ReportStreamFilter = emptyList(),
+    val routingFilter: ReportStreamFilter = emptyList(),
+    val processingModeFilter: ReportStreamFilter = emptyList(),
     val reverseTheQualityFilter: Boolean = false,
     val deidentify: Boolean = false,
     val timing: Timing? = null,
@@ -60,6 +65,8 @@ open class Receiver(
         copy.translation,
         copy.jurisdictionalFilter,
         copy.qualityFilter,
+        copy.routingFilter,
+        copy.processingModeFilter,
         copy.reverseTheQualityFilter,
         copy.deidentify,
         copy.timing,