Added check for last modified time to settings CLI (#3403)

* Added check for last modified time to settings CLI

prime-router/build.gradle.kts

@@ -110,8 +110,9 @@ tasks.clean {
     delete("target")
     // clean up all the old event files in the SOAP set up
     doLast {
-        val eventsDir = File("../.environment/soap_service/soap/event/v1/")
-        if (eventsDir.exists()) {
+        val eventsDir = File("../.environment/soap_service/soap/event/v1")
+        if (eventsDir.exists() && eventsDir.isDirectory && eventsDir.listFiles().isNotEmpty()) {
+            // Note FileUtils does not like when the folder is empty.
             FileUtils.listFiles(eventsDir, arrayOf("event"), true).forEach {
                 it.delete()
             }

prime-router/docs/getting-started/faster-development.md

@@ -21,7 +21,7 @@ a one time procedure and only needs to be run at workstation startup or when you
 
 ```bash
 docker-compose -f docker-compose.build.yml up --detach
-docker-compose up --scale prime_dev=0 --scale settings=0 --scale web_receiver=0 --detach
+docker-compose up --scale prime_dev=0 --detach
 ```
 
 ### Running the Azure functions

prime-router/src/main/kotlin/azure/HttpUtilities.kt

@@ -26,6 +26,11 @@ class HttpUtilities {
         const val watersApi = "/api/waters"
         const val tokenApi = "/api/token"
 
+        /**
+         * Last modified time header value formatter.
+         */
+        val lastModifiedFormatter: DateTimeFormatter = DateTimeFormatter.RFC_1123_DATE_TIME
+
         fun okResponse(
             request: HttpRequestMessage<String?>,
             responseBody: String,
@@ -151,10 +156,10 @@ class HttpUtilities {
         ) {
             if (lastModified == null) return
             // https://datatracker.ietf.org/doc/html/rfc7232#section-2.2 defines this header format
-            val lastModifiedFormatter = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss")
-            // Convert to GMT timezone
+
+            // Convert to UTC timezone
             val lastModifiedGMT = OffsetDateTime.ofInstant(lastModified.toInstant(), ZoneOffset.UTC)
-            val lastModifiedFormatted = "${lastModifiedGMT.format(lastModifiedFormatter)} GMT"
+            val lastModifiedFormatted = lastModifiedGMT.format(lastModifiedFormatter)
             builder.header(HttpHeaders.LAST_MODIFIED, lastModifiedFormatted)
         }
 

prime-router/src/main/kotlin/cli/LookupTableCommands.kt

@@ -790,6 +790,15 @@ class LookupTableLoadAllCommand : GenericLookupTableCommand(
     private val connRetries by option("-r", "--retries", help = "Number of seconds to retry waiting for the API")
         .int().default(30)
 
+    /**
+     * Number of connection retries.
+     */
+    private val checkLastModified by option(
+        "--check-last-modified",
+        help = "Update settings only if input file is newer"
+    )
+        .flag(default = false)
+
     /**
      * The reference to the table creator command.
      */
@@ -803,9 +812,11 @@ class LookupTableLoadAllCommand : GenericLookupTableCommand(
         CommandUtilities.waitForApi(environment, connRetries)
 
         // Get the list of current tables to only update or create new ones.
-        val tableUpdateTimes = LookupTableEndpointUtilities(environment).fetchList().map {
-            it.tableName to it.createdAt
-        }.toMap()
+        val tableUpdateTimes = if (checkLastModified)
+            LookupTableEndpointUtilities(environment).fetchList().map {
+                it.tableName to it.createdAt
+            }.toMap()
+        else emptyMap()
 
         // Loop through all the files
         val files = try {
@@ -819,7 +830,7 @@ class LookupTableLoadAllCommand : GenericLookupTableCommand(
             var needToLoad = true
             // If we have a table in the database then only update it if the last modified time of the file is 
             // greater than the created time in the database.
-            if (tableUpdateTimes.contains(tableName) && tableUpdateTimes[tableName] != null) {
+            if (checkLastModified && tableUpdateTimes.contains(tableName) && tableUpdateTimes[tableName] != null) {
                 val fileUpdatedTime = Instant.ofEpochMilli(it.lastModified())
                 if (!fileUpdatedTime.isAfter(tableUpdateTimes[tableName]!!.toInstant())) {
                     needToLoad = false

prime-router/src/main/kotlin/cli/SettingCommands.kt

@@ -16,6 +16,7 @@ import com.github.ajalt.clikt.parameters.options.flag
 import com.github.ajalt.clikt.parameters.options.option
 import com.github.ajalt.clikt.parameters.options.required
 import com.github.ajalt.clikt.parameters.types.choice
+import com.github.ajalt.clikt.parameters.types.file
 import com.github.ajalt.clikt.parameters.types.inputStream
 import com.github.ajalt.clikt.parameters.types.int
 import com.github.ajalt.clikt.parameters.types.outputStream
@@ -27,16 +28,21 @@ import com.github.kittinunf.fuel.core.extensions.authentication
 import com.github.kittinunf.fuel.json.FuelJson
 import com.github.kittinunf.fuel.json.responseJson
 import com.github.kittinunf.result.Result
+import com.google.common.net.HttpHeaders
 import gov.cdc.prime.router.DeepOrganization
 import gov.cdc.prime.router.Organization
 import gov.cdc.prime.router.Receiver
 import gov.cdc.prime.router.Sender
+import gov.cdc.prime.router.azure.HttpUtilities
 import gov.cdc.prime.router.azure.OrganizationAPI
 import gov.cdc.prime.router.azure.ReceiverAPI
 import gov.cdc.prime.router.azure.SenderAPI
 import gov.cdc.prime.router.common.Environment
 import org.apache.http.HttpStatus
+import java.io.File
 import java.io.InputStream
+import java.time.OffsetDateTime
+import java.time.format.DateTimeParseException
 
 private const val apiPath = "/api/settings"
 private const val dummyAccessToken = "dummy"
@@ -192,13 +198,25 @@ abstract class SettingCommand(
         }
     }
 
-    fun readInput(): String {
-        if (inStream == null) abort("Missing input file")
-        val input = String(inStream!!.readAllBytes())
+    /**
+     * Read the contents of an [inputStream].
+     * @return the file contents
+     */
+    fun readInput(inputStream: InputStream? = inStream): String {
+        if (inputStream == null) abort("Missing input file")
+        val input = String(inputStream.readAllBytes())
         if (input.isBlank()) abort("Blank input")
         return input
     }
 
+    /**
+     * Read the contents [file].
+     * @return the file contents
+     */
+    fun readInput(file: File): String {
+        return readInput(file.inputStream())
+    }
+
     fun writeOutput(output: String) {
         outStream.write(output.toByteArray())
     }
@@ -518,15 +536,27 @@ class PutMultipleSettings : SettingCommand(
     help = "set all settings from a 'organizations.yml' file"
 ) {
 
-    override val inStream by option("-i", "--input", help = "Input from file", metavar = "<file>")
-        .inputStream()
+    /**
+     * Input file with the settings.
+     */
+    private val inputFile by option("-i", "--input", help = "Input from file", metavar = "<file>")
+        .file(true, mustBeReadable = true).required()
 
     /**
      * Number of connection retries.
      */
     private val connRetries by option("-r", "--retries", help = "Number of seconds to retry waiting for the API")
         .int().default(30)
 
+    /**
+     * Number of connection retries.
+     */
+    private val checkLastModified by option(
+        "--check-last-modified",
+        help = "Update settings only if input file is newer"
+    )
+        .flag(default = false)
+
     override fun run() {
         val environment = Environment.get(env)
         val accessToken = getAccessToken(environment)
@@ -535,13 +565,45 @@ class PutMultipleSettings : SettingCommand(
         TermUi.echo("Waiting for the API at ${environment.url} to be available...")
         CommandUtilities.waitForApi(environment, connRetries)
 
-        val results = putAll(environment, accessToken)
-        val output = "${results.joinToString("\n")}\n"
-        writeOutput(output)
+        if (!checkLastModified || (checkLastModified && isFileUpdated(environment))) {
+            TermUi.echo("Loading settings from ${inputFile.absolutePath}...")
+            val results = putAll(environment, accessToken)
+            val output = "${results.joinToString("\n")}\n"
+            writeOutput(output)
+        } else {
+            TermUi.echo("No new updates found for settings.")
+        }
+    }
+
+    /**
+     * Check if the settings from a file are newer than the data stored in the database for the
+     * given [environment].
+     * @return true if the file settings are newer or there is nothing in the database, false otherwise
+     */
+    private fun isFileUpdated(environment: Environment): Boolean {
+        val url = formPath(environment, Operation.LIST, SettingType.ORG, "")
+        val (_, response, result) = Fuel.head(url).authentication()
+            .bearer(getAccessToken(environment)).response()
+        return when (result) {
+            is Result.Success -> {
+                if (response[HttpHeaders.LAST_MODIFIED].isNotEmpty()) {
+                    try {
+                        val apiModifiedTime = OffsetDateTime.parse(
+                            response[HttpHeaders.LAST_MODIFIED].first(),
+                            HttpUtilities.lastModifiedFormatter
+                        )
+                        apiModifiedTime.toInstant().toEpochMilli() < inputFile.lastModified()
+                    } catch (e: DateTimeParseException) {
+                        error("Unable to decode last modified data from API call. $e")
+                    }
+                } else true // We have no last modified time, which means the DB is empty
+            }
+            else -> error("Unable to fetch settings last update time from API.  $result")
+        }
     }
 
     private fun putAll(environment: Environment, accessToken: String): List<String> {
-        val deepOrgs = readYaml()
+        val deepOrgs = readYaml(inputFile)
         val results = mutableListOf<String>()
         // Put orgs
         deepOrgs.forEach { deepOrg ->
@@ -562,8 +624,12 @@ class PutMultipleSettings : SettingCommand(
         return results
     }
 
-    private fun readYaml(): List<DeepOrganization> {
-        val input = readInput()
+    /**
+     * Read the settings from a YAML file.
+     * @return the settings
+     */
+    private fun readYaml(file: File): List<DeepOrganization> {
+        val input = readInput(file.inputStream())
         return yamlMapper.readValue(input)
     }
 }

prime-router/start_func.sh

@@ -30,13 +30,15 @@ function load_config() {
   top_dir=$function_folder/../..
   test_config_dir=$top_dir/resources/test
   fatjar=$top_dir/libs/prime-router-0.1-SNAPSHOT-all.jar
-  java -jar $fatjar lookuptables loadall -d $test_config_dir/metadata/tables -r 60 | awk '{print "[LOAD TABLES] " $0}'
+  echo "Loading lookup tables..."
+  java -jar $fatjar lookuptables loadall -d $test_config_dir/metadata/tables -r 60 --check-last-modified
   # Note the settings require the full metadata catalog to be in place, so run last
-  java -jar $fatjar multiple-settings set -i $function_folder/settings/organizations.yml -r 60 | awk '{print "[LOAD SETTINGS] " $0}'
+  echo "Loading organization settings..."
+  java -jar $fatjar multiple-settings set -i $function_folder/settings/organizations.yml -r 60 --check-last-modified
 }
 
 # Load the configuration in the background.  It will wait for the API to start the loading.
-load_config &
+load_config | awk '{print "[LOAD CONFIG] " $0}' &
 
 # Run the functions
 func host start --cors http://localhost:8090,http://localhost:3000 --language-worker -- "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005"