optimize hikari cp for scale-zero connection pool (neon db evaluation…

…), refactor profiles in application properties, refactor debug log statements

kotlin/config/application.properties.tmpl

@@ -23,3 +23,8 @@ app.kafka.brokers=<list-of-brokers>
 app.kafka.topic-prefix=<optional-prefix>
 app.kafka.sasl-username=<userid>
 app.kafka.sasl-password=<password
+
+# enable to overwrite datasource properties
+# spring.datasource.username=<user>
+# spring.datasource.password=<password>
+# spring.datasource.url=jdbc:postgresql://<url>/<db>?sslmode=require

kotlin/dbimport.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+
 #set -e -o pipefail
 if [ -z "$PGDATA" ]; then echo "PGDATA not set"; exit 1; fi
 if ! hash pg_ctl 2>/dev/null; then echo psql client tools such as pg_ctl are not installed; exit 2; fi
@@ -78,23 +79,24 @@ pg_restore --use-list "$(dirname $local_dump)/pg_restore_list" \
 { set +x; } 2>/dev/null
 logit "Backup finished, running select check on $local_db_dev ($local_db_test remains empty)"
 logit "Most recent backup may be from last night, run 'appctl backup-db' for a fresh one!"
+
 psql -U $local_role -d $local_db_dev <<-EOF
-SELECT table_name,pg_size_pretty( pg_total_relation_size(quote_ident(table_name)))
-FROM information_schema.tables WHERE table_schema = 'public'
-ORDER BY pg_total_relation_size(quote_ident(table_name)) DESC
+  SELECT table_name,pg_size_pretty( pg_total_relation_size(quote_ident(table_name)))
+  FROM information_schema.tables WHERE table_schema = 'public'
+  ORDER BY pg_total_relation_size(quote_ident(table_name)) DESC
 EOF
-psqlexit=$?
-if [ $psqlexit -ne 0 ]; then
-  echo "psql failed with exit code $psqlexit"
-  exit $psqlexit;
+psql_exit=$?
+if [ psql_exit -ne 0 ]; then
+  echo "psql failed with exit code psql_exit"
+  exit psql_exit;
 fi
 
 logit "Syncing s3://${bucket_name}/imagine with ${bucket_name}-dev"
 # AWS S3 SYNC exclude patterns: https://stackoverflow.com/a/32394703/4292075
-# *: Matches everything
-# ?: Matches any single character
-# [sequence]: Matches any character in sequence
-# [!sequence]: Matches any character not in sequence
+#  *: Matches everything
+#  ?: Matches any single character
+#  [sequence]: Matches any character in sequence
+#  [!sequence]: Matches any character not in sequence
 # Storage Classes: https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/s3/types#ObjectStorageClass
 aws s3 sync s3://${bucket_name}/imagine s3://${bucket_name}-dev/imagine --delete \
     --exclude '*songs/A*' --exclude '*songs/C*' --exclude '*songs/S*' --exclude '*songs/E*' \

kotlin/src/main/kotlin/net/timafe/angkor/repo/DatabaseHealthContributor.kt

@@ -1,5 +1,7 @@
 package net.timafe.angkor.repo
 
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
 import org.springframework.boot.actuate.health.Health
 import org.springframework.boot.actuate.health.HealthContributor
 import org.springframework.boot.actuate.health.HealthIndicator
@@ -23,8 +25,11 @@ class DatabaseHealthContributor(
 
 ) : HealthIndicator, HealthContributor {
 
+    private val log: Logger = LoggerFactory.getLogger(this.javaClass)
+
     override fun health(): Health? {
         try {
+            log.trace("DB Health Check")
             ds.connection.use { conn ->
                 val stmt: Statement = conn.createStatement()
                 stmt.execute("select count(*) from location")

kotlin/src/main/kotlin/net/timafe/angkor/service/DishService.kt

@@ -34,7 +34,7 @@ class DishService(
      */
     @CacheEvict(cacheNames = [TagRepository.TAGS_FOR_DISHES_CACHE], allEntries = true)
     override fun save(item: Dish): Dish {
-        log.debug("save${entityType()}: $item")
+        log.debug("save{}: {}", entityType(), item)
         val autoTags = mutableListOf<String>()
         val area = getArea(item.areaCode)
         if (area?.adjectival != null) autoTags.add(area.adjectival!!)

kotlin/src/main/kotlin/net/timafe/angkor/service/EventService.kt

@@ -103,7 +103,7 @@ class EventService(
         // by default source applies to the entire app (e.g. angkor-api)
         event.source = event.source ?: env.getProperty("spring.application.name")
         if (kafkaEnabled()) {
-            log.debug("$logPrefix Publish event '$event' to $topicStr async=${Thread.currentThread().name}")
+            log.debug("{} Publish event '{}' to {} async={}", logPrefix, event, topicStr, Thread.currentThread().name)
             try {
                 val eventStr = objectMapper
                     .writer()
@@ -133,9 +133,11 @@ class EventService(
         }
     }
 
+    // CAUTION: each call of consumeMessages requires an active DB Connection from the Pool
+    // Value increased to 300000 (5min) to increase the time that hikari cp can be scaled to 0
     // durations are in milliseconds. also supports ${my.delay.property} (escape with \ or kotlin compiler complains)
     // 600000 = 10 Minutes make sure @EnableScheduling is active in AsyncConfig 600000 = 10 min, 3600000 = 1h
-    @Scheduled(fixedRateString = "120000", initialDelay = 20000)
+    @Scheduled(fixedRateString = "300000", initialDelay = 20000)
     @Transactional
     fun consumeMessages() {
         // @Scheduled runs without Auth Context, so we use a special ServiceAccountToken here
@@ -146,7 +148,7 @@ class EventService(
         // https://www.oreilly.com/library/view/kafka-the-definitive/9781491936153/ch04.html
         val consumer: KafkaConsumer<String, String> = KafkaConsumer<String, String>(this.consumerProps)
         val topics = listOf("imagine", "audit", "system", "app").map { "${appProps.kafka.topicPrefix}$it" }
-        log.trace(" $logPrefix I'm here to consume new Kafka Messages from topics $topics")
+        log.trace(" {} I'm here to consume new Kafka Messages from topics {}", logPrefix, topics)
         consumer.subscribe(topics)
         var (received, persisted) = listOf(0, 0)
         val records = consumer.poll(Duration.ofMillis(10L * 1000))
@@ -226,7 +228,7 @@ class EventService(
                     log.warn("${logPrefix()} Could not convert messageId $mid to UUID, generating new one")
                     UUID.randomUUID()
                 } else {
-                    log.debug("${logPrefix()} using messageId from header $midUUID")
+                    log.debug("{} using messageId from header {}", logPrefix(), midUUID)
                     midUUID
                 }
             }

kotlin/src/main/resources/application.yml

@@ -3,7 +3,7 @@
 
 # Application Properties, see also class n.t.a.config.AppProperties
 app:
-  api-token: sieam-reap # only for local testing
+  api-token: siem-reap # only for local testing
   admin-mail: angkor-admin@localhost # local testing, may be overwritten by config/application.properties
   osm-api-base-url: https://nominatim.openstreetmap.org
   # 600s = 10 min, 3600s = 1h, 86400s = 1day (using seconds, 43200 = 12h default is millis)
@@ -44,6 +44,10 @@ logging:
     org.springframework.context.support.PostProcessorRegistrationDelegate: WARN
     # 2024-02 on demand "hidden" logger: https://stackoverflow.com/questions/3686196/how-to-show-all-available-routes-in-spring
     # _org.springframework.web.servlet.HandlerMapping.Mappings: debug
+
+    # Debug HikariCP behaviour https://stackoverflow.com/a/60778768/4292075
+    # com.zaxxer.hikari.HikariConfig: DEBUG
+    com.zaxxer.hikari: TRACE
   pattern:
     # Spring Boot - Customizing Console Logging Format
     # https://www.logicbig.com/tutorials/spring-framework/spring-boot/logging-console-pattern.html
@@ -90,20 +94,27 @@ server:
 spring:
   application:
     name: angkor-api
+
   datasource:
     # set env SPRING_DATASOURCE_URL= jdbc:postgres://host:5432/dbname, same with USERNAME and PASSWORD
     driver-class-name: org.postgresql.Driver
+    # Hikari Config Params https://github.com/brettwooldridge/HikariCP?tab=readme-ov-file#gear-configuration-knobs-baby
     hikari:
       # ElephantSQL only supports 5 concurrent connections, so we use small pool sizes
       maximum-pool-size: 3 # SPRING_DATASOURCE_HIKARI_MAXIMUM_POOL_SIZE: "2"
-      minimum-idle: 2 # SPRING_DATASOURCE_HIKARI_MINIMUM_IDLE: "2"
+      # This property controls the minimum number of idle connections that HikariCP tries to maintain in the pool.
+      minimum-idle: 0 # 2 # SPRING_DATASOURCE_HIKARI_MINIMUM_IDLE: "2"
+      # This property controls the maximum amount of time that a connection is allowed to sit idle in the pool.
+      idle-timeout: 46000 # aggressive 45s, Default is: 600000 (10 minutes),
+
   jpa:
     database: postgresql
     generate-ddl: false # we rely on flyway
     hibernate:
       ddl-auto: none # none, validate, update, create-drop.
     open-in-view: false # to disable database queries may be performed during view rendering warning
     show-sql: false # only set show sql to true on demand, please
+
   kafka:
     # global bootstrap-servers and security config, can be overwritten in consumer / producer
     security:
@@ -127,13 +138,37 @@ spring:
         # use SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG to configure via environment
         jaas:
           config: org.apache.kafka.common.security.plain.PlainLoginModule required username="<set-user>" password="<set-pw>";
+
+  mail:
+    host: email-smtp.eu-central-1.amazonaws.com
+    # standard mail props as per to https://javaee.github.io/javamail/docs/api/com/sun/mail/smtp/package-summary.html
+    properties:
+      # usually we want to enforce tls, but not for test (greenmail is on localhost w/o tls)
+      mail.smtp.auth: true
+      mail.smtp.starttls.enable: true
+      mail.smtp.starttls.required: true
+      # https://docs.aws.amazon.com/ses/latest/dg/smtp-connect.html STARTTLS port =
+      # To set up a STARTTLS connection, the SMTP client connects to the Amazon SES SMTP endpoint on port 25, 587, or 2587,
+      # issues an EHLO (SMTP extended hello) command, and waits for the server to announce that it supports the
+      # STARTTLS SMTP extension. CAUTION: do not use ports 465 or 2465 here, they are for the TLS Wrapper connection
+      mail.smtp.port: 2587
+      ## various timeouts
+      mail.smtp.timeout:  5000
+      mail.smtp.connectiontimeout: 5000
+      mail.smtp.writetimeout: 5000
+      # default from address
+      mail.smtp.from: angkor-thom@localhost
+
   main:
     allow-bean-definition-overriding: true
     banner-mode: "off"
+
   mvc:
     converters:
       preferred-json-mapper: jackson
+
   security:
+    # https://docs.spring.io/spring-security/reference/servlet/oauth2/login/core.html#oauth2login-boot-property-mappings
     oauth2:
       client:
         registration:
@@ -143,7 +178,8 @@ spring:
             #  client-secret: please_overwrite_via_env
             scope: openid
             provider: cognito
-            redirectUriTemplate: http://localhost:8080/login/oauth2/code/cognito
+            # 2024-09-24 redirectUriTemplate has been removed in favor of redirectUri https://github.com/spring-projects/spring-security/issues/8830
+            redirectUri: http://localhost:8080/login/oauth2/code/cognito
         provider:
           cognito:
             # 2023-01-19 seems previous attribute "cognito:username" meanwhile arrives only as "username" in
@@ -152,6 +188,7 @@ spring:
             user-name-attribute: username
             # issuer-uri: please_overwrite_via_env
             # redirectUriTemplate: http://localhost:8080/login/oauth2/code/cognito # needed?
+
   servlet:
     multipart:
       max-file-size: 10MB # 1 MB is default
@@ -169,48 +206,34 @@ spring:
       thread-name-prefix: angkor-scheduling-
       pool:
         size: 2
+
   sql:
     init:
       platform: postgres
 
+
 ---
-# spring default profile, if no specific profile activated, suitable for local dev
+# spring 'default' profile overwrites, if no specific profile activated, suitable for local dev
 spring:
   config:
     activate:
       on-profile: default
+
   datasource:
     url: jdbc:postgresql://localhost:5432/angkor_dev
     username: angkor_dev
     password:
-  mail:
-    host: email-smtp.eu-central-1.amazonaws.com
-    # standard mail props as per to https://javaee.github.io/javamail/docs/api/com/sun/mail/smtp/package-summary.html
-    properties:
-      # usually we want to enforce tls, but not for test (greenmail is on localhost w/o tls)
-      mail.smtp.auth: true
-      mail.smtp.starttls.enable: true
-      mail.smtp.starttls.required: true
-      # https://docs.aws.amazon.com/ses/latest/dg/smtp-connect.html STARTTLS port =
-      # To set up a STARTTLS connection, the SMTP client connects to the Amazon SES SMTP endpoint on port 25, 587, or 2587,
-      # issues an EHLO (SMTP extended hello) command, and waits for the server to announce that it supports the
-      # STARTTLS SMTP extension. CAUTION: do not use ports 465 or 2465 here, they are for the TLS Wrapper connection
-      mail.smtp.port: 2587
-      ## various timeouts
-      mail.smtp.timeout:  5000
-      mail.smtp.connectiontimeout: 5000
-      mail.smtp.writetimeout: 5000
-      # default from address
-      mail.smtp.from: angkor-thom@localhost
+
 
 ---
-# overwrites for spring 'prod' profile, none-sensitive test specific values only
+# spring 'prod' profile overwrites, none-sensitive test specific values only
 # for sensitive values see config/application.properties, which is NOT under version control
 # alternatively, sensitive prod values may be injected via environment
 spring:
   config:
     activate:
       on-profile: prod
+
   flyway:
     clean-disabled: true
   jackson:
@@ -221,9 +244,10 @@ server:
   error:
     include-message: never # disable for prod to avoid leaking of info to client (default for spring >= 2.3)
 
+
 ---
-# overwrites for spring 'test' profile, none-sensitive test specific values only
-# for sensitive values see src/test/resources/application-test.properties, which is NOT under version control
+# spring 'test' profile overwrites, use for none-sensitive test specific values only
+# for sensitive values, see src/test/resources/application-test.properties, which is NOT under version control
 spring:
   config:
     activate: