From 9c97f69d30e5f1b2f96fa34bff169ec0a3ccfbda Mon Sep 17 00:00:00 2001 From: Till Kuhn Date: Tue, 24 Sep 2024 20:07:20 +0200 Subject: [PATCH] optimize hikari cp for scale-zero connection pool (neon db evaluation), refactor profiles in application properties, refactor debug log statements --- kotlin/config/application.properties.tmpl | 5 ++ kotlin/dbimport.sh | 24 +++--- .../angkor/repo/DatabaseHealthContributor.kt | 5 ++ .../net/timafe/angkor/service/DishService.kt | 2 +- .../net/timafe/angkor/service/EventService.kt | 10 ++- kotlin/src/main/resources/application.yml | 76 ++++++++++++------- 6 files changed, 80 insertions(+), 42 deletions(-) diff --git a/kotlin/config/application.properties.tmpl b/kotlin/config/application.properties.tmpl index 72179ffce..7240e003f 100644 --- a/kotlin/config/application.properties.tmpl +++ b/kotlin/config/application.properties.tmpl @@ -23,3 +23,8 @@ app.kafka.brokers= app.kafka.topic-prefix= app.kafka.sasl-username= app.kafka.sasl-password= +# spring.datasource.password= +# spring.datasource.url=jdbc:postgresql:///?sslmode=require diff --git a/kotlin/dbimport.sh b/kotlin/dbimport.sh index d65e16b37..01db61000 100755 --- a/kotlin/dbimport.sh +++ b/kotlin/dbimport.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash + #set -e -o pipefail if [ -z "$PGDATA" ]; then echo "PGDATA not set"; exit 1; fi if ! hash pg_ctl 2>/dev/null; then echo psql client tools such as pg_ctl are not installed; exit 2; fi @@ -78,23 +79,24 @@ pg_restore --use-list "$(dirname $local_dump)/pg_restore_list" \ { set +x; } 2>/dev/null logit "Backup finished, running select check on $local_db_dev ($local_db_test remains empty)" logit "Most recent backup may be from last night, run 'appctl backup-db' for a fresh one!" + psql -U $local_role -d $local_db_dev <<-EOF -SELECT table_name,pg_size_pretty( pg_total_relation_size(quote_ident(table_name))) -FROM information_schema.tables WHERE table_schema = 'public' -ORDER BY pg_total_relation_size(quote_ident(table_name)) DESC + SELECT table_name,pg_size_pretty( pg_total_relation_size(quote_ident(table_name))) + FROM information_schema.tables WHERE table_schema = 'public' + ORDER BY pg_total_relation_size(quote_ident(table_name)) DESC EOF -psqlexit=$? -if [ $psqlexit -ne 0 ]; then - echo "psql failed with exit code $psqlexit" - exit $psqlexit; +psql_exit=$? +if [ psql_exit -ne 0 ]; then + echo "psql failed with exit code psql_exit" + exit psql_exit; fi logit "Syncing s3://${bucket_name}/imagine with ${bucket_name}-dev" # AWS S3 SYNC exclude patterns: https://stackoverflow.com/a/32394703/4292075 -# *: Matches everything -# ?: Matches any single character -# [sequence]: Matches any character in sequence -# [!sequence]: Matches any character not in sequence +# *: Matches everything +# ?: Matches any single character +# [sequence]: Matches any character in sequence +# [!sequence]: Matches any character not in sequence # Storage Classes: https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/s3/types#ObjectStorageClass aws s3 sync s3://${bucket_name}/imagine s3://${bucket_name}-dev/imagine --delete \ --exclude '*songs/A*' --exclude '*songs/C*' --exclude '*songs/S*' --exclude '*songs/E*' \ diff --git a/kotlin/src/main/kotlin/net/timafe/angkor/repo/DatabaseHealthContributor.kt b/kotlin/src/main/kotlin/net/timafe/angkor/repo/DatabaseHealthContributor.kt index eeb2fa257..509b3f499 100644 --- a/kotlin/src/main/kotlin/net/timafe/angkor/repo/DatabaseHealthContributor.kt +++ b/kotlin/src/main/kotlin/net/timafe/angkor/repo/DatabaseHealthContributor.kt @@ -1,5 +1,7 @@ package net.timafe.angkor.repo +import org.slf4j.Logger +import org.slf4j.LoggerFactory import org.springframework.boot.actuate.health.Health import org.springframework.boot.actuate.health.HealthContributor import org.springframework.boot.actuate.health.HealthIndicator @@ -23,8 +25,11 @@ class DatabaseHealthContributor( ) : HealthIndicator, HealthContributor { + private val log: Logger = LoggerFactory.getLogger(this.javaClass) + override fun health(): Health? { try { + log.trace("DB Health Check") ds.connection.use { conn -> val stmt: Statement = conn.createStatement() stmt.execute("select count(*) from location") diff --git a/kotlin/src/main/kotlin/net/timafe/angkor/service/DishService.kt b/kotlin/src/main/kotlin/net/timafe/angkor/service/DishService.kt index b3d79ecd6..2d171c6f9 100644 --- a/kotlin/src/main/kotlin/net/timafe/angkor/service/DishService.kt +++ b/kotlin/src/main/kotlin/net/timafe/angkor/service/DishService.kt @@ -34,7 +34,7 @@ class DishService( */ @CacheEvict(cacheNames = [TagRepository.TAGS_FOR_DISHES_CACHE], allEntries = true) override fun save(item: Dish): Dish { - log.debug("save${entityType()}: $item") + log.debug("save{}: {}", entityType(), item) val autoTags = mutableListOf() val area = getArea(item.areaCode) if (area?.adjectival != null) autoTags.add(area.adjectival!!) diff --git a/kotlin/src/main/kotlin/net/timafe/angkor/service/EventService.kt b/kotlin/src/main/kotlin/net/timafe/angkor/service/EventService.kt index 9f4bb4b77..3ec5d0d12 100644 --- a/kotlin/src/main/kotlin/net/timafe/angkor/service/EventService.kt +++ b/kotlin/src/main/kotlin/net/timafe/angkor/service/EventService.kt @@ -103,7 +103,7 @@ class EventService( // by default source applies to the entire app (e.g. angkor-api) event.source = event.source ?: env.getProperty("spring.application.name") if (kafkaEnabled()) { - log.debug("$logPrefix Publish event '$event' to $topicStr async=${Thread.currentThread().name}") + log.debug("{} Publish event '{}' to {} async={}", logPrefix, event, topicStr, Thread.currentThread().name) try { val eventStr = objectMapper .writer() @@ -133,9 +133,11 @@ class EventService( } } + // CAUTION: each call of consumeMessages requires an active DB Connection from the Pool + // Value increased to 300000 (5min) to increase the time that hikari cp can be scaled to 0 // durations are in milliseconds. also supports ${my.delay.property} (escape with \ or kotlin compiler complains) // 600000 = 10 Minutes make sure @EnableScheduling is active in AsyncConfig 600000 = 10 min, 3600000 = 1h - @Scheduled(fixedRateString = "120000", initialDelay = 20000) + @Scheduled(fixedRateString = "300000", initialDelay = 20000) @Transactional fun consumeMessages() { // @Scheduled runs without Auth Context, so we use a special ServiceAccountToken here @@ -146,7 +148,7 @@ class EventService( // https://www.oreilly.com/library/view/kafka-the-definitive/9781491936153/ch04.html val consumer: KafkaConsumer = KafkaConsumer(this.consumerProps) val topics = listOf("imagine", "audit", "system", "app").map { "${appProps.kafka.topicPrefix}$it" } - log.trace(" $logPrefix I'm here to consume new Kafka Messages from topics $topics") + log.trace(" {} I'm here to consume new Kafka Messages from topics {}", logPrefix, topics) consumer.subscribe(topics) var (received, persisted) = listOf(0, 0) val records = consumer.poll(Duration.ofMillis(10L * 1000)) @@ -226,7 +228,7 @@ class EventService( log.warn("${logPrefix()} Could not convert messageId $mid to UUID, generating new one") UUID.randomUUID() } else { - log.debug("${logPrefix()} using messageId from header $midUUID") + log.debug("{} using messageId from header {}", logPrefix(), midUUID) midUUID } } diff --git a/kotlin/src/main/resources/application.yml b/kotlin/src/main/resources/application.yml index cc40ab36a..8743686ef 100644 --- a/kotlin/src/main/resources/application.yml +++ b/kotlin/src/main/resources/application.yml @@ -3,7 +3,7 @@ # Application Properties, see also class n.t.a.config.AppProperties app: - api-token: sieam-reap # only for local testing + api-token: siem-reap # only for local testing admin-mail: angkor-admin@localhost # local testing, may be overwritten by config/application.properties osm-api-base-url: https://nominatim.openstreetmap.org # 600s = 10 min, 3600s = 1h, 86400s = 1day (using seconds, 43200 = 12h default is millis) @@ -44,6 +44,10 @@ logging: org.springframework.context.support.PostProcessorRegistrationDelegate: WARN # 2024-02 on demand "hidden" logger: https://stackoverflow.com/questions/3686196/how-to-show-all-available-routes-in-spring # _org.springframework.web.servlet.HandlerMapping.Mappings: debug + + # Debug HikariCP behaviour https://stackoverflow.com/a/60778768/4292075 + # com.zaxxer.hikari.HikariConfig: DEBUG + com.zaxxer.hikari: TRACE pattern: # Spring Boot - Customizing Console Logging Format # https://www.logicbig.com/tutorials/spring-framework/spring-boot/logging-console-pattern.html @@ -90,13 +94,19 @@ server: spring: application: name: angkor-api + datasource: # set env SPRING_DATASOURCE_URL= jdbc:postgres://host:5432/dbname, same with USERNAME and PASSWORD driver-class-name: org.postgresql.Driver + # Hikari Config Params https://github.com/brettwooldridge/HikariCP?tab=readme-ov-file#gear-configuration-knobs-baby hikari: # ElephantSQL only supports 5 concurrent connections, so we use small pool sizes maximum-pool-size: 3 # SPRING_DATASOURCE_HIKARI_MAXIMUM_POOL_SIZE: "2" - minimum-idle: 2 # SPRING_DATASOURCE_HIKARI_MINIMUM_IDLE: "2" + # This property controls the minimum number of idle connections that HikariCP tries to maintain in the pool. + minimum-idle: 0 # 2 # SPRING_DATASOURCE_HIKARI_MINIMUM_IDLE: "2" + # This property controls the maximum amount of time that a connection is allowed to sit idle in the pool. + idle-timeout: 46000 # aggressive 45s, Default is: 600000 (10 minutes), + jpa: database: postgresql generate-ddl: false # we rely on flyway @@ -104,6 +114,7 @@ spring: ddl-auto: none # none, validate, update, create-drop. open-in-view: false # to disable database queries may be performed during view rendering warning show-sql: false # only set show sql to true on demand, please + kafka: # global bootstrap-servers and security config, can be overwritten in consumer / producer security: @@ -127,13 +138,37 @@ spring: # use SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG to configure via environment jaas: config: org.apache.kafka.common.security.plain.PlainLoginModule required username="" password=""; + + mail: + host: email-smtp.eu-central-1.amazonaws.com + # standard mail props as per to https://javaee.github.io/javamail/docs/api/com/sun/mail/smtp/package-summary.html + properties: + # usually we want to enforce tls, but not for test (greenmail is on localhost w/o tls) + mail.smtp.auth: true + mail.smtp.starttls.enable: true + mail.smtp.starttls.required: true + # https://docs.aws.amazon.com/ses/latest/dg/smtp-connect.html STARTTLS port = + # To set up a STARTTLS connection, the SMTP client connects to the Amazon SES SMTP endpoint on port 25, 587, or 2587, + # issues an EHLO (SMTP extended hello) command, and waits for the server to announce that it supports the + # STARTTLS SMTP extension. CAUTION: do not use ports 465 or 2465 here, they are for the TLS Wrapper connection + mail.smtp.port: 2587 + ## various timeouts + mail.smtp.timeout: 5000 + mail.smtp.connectiontimeout: 5000 + mail.smtp.writetimeout: 5000 + # default from address + mail.smtp.from: angkor-thom@localhost + main: allow-bean-definition-overriding: true banner-mode: "off" + mvc: converters: preferred-json-mapper: jackson + security: + # https://docs.spring.io/spring-security/reference/servlet/oauth2/login/core.html#oauth2login-boot-property-mappings oauth2: client: registration: @@ -143,7 +178,8 @@ spring: # client-secret: please_overwrite_via_env scope: openid provider: cognito - redirectUriTemplate: http://localhost:8080/login/oauth2/code/cognito + # 2024-09-24 redirectUriTemplate has been removed in favor of redirectUri https://github.com/spring-projects/spring-security/issues/8830 + redirectUri: http://localhost:8080/login/oauth2/code/cognito provider: cognito: # 2023-01-19 seems previous attribute "cognito:username" meanwhile arrives only as "username" in @@ -152,6 +188,7 @@ spring: user-name-attribute: username # issuer-uri: please_overwrite_via_env # redirectUriTemplate: http://localhost:8080/login/oauth2/code/cognito # needed? + servlet: multipart: max-file-size: 10MB # 1 MB is default @@ -169,48 +206,34 @@ spring: thread-name-prefix: angkor-scheduling- pool: size: 2 + sql: init: platform: postgres + --- -# spring default profile, if no specific profile activated, suitable for local dev +# spring 'default' profile overwrites, if no specific profile activated, suitable for local dev spring: config: activate: on-profile: default + datasource: url: jdbc:postgresql://localhost:5432/angkor_dev username: angkor_dev password: - mail: - host: email-smtp.eu-central-1.amazonaws.com - # standard mail props as per to https://javaee.github.io/javamail/docs/api/com/sun/mail/smtp/package-summary.html - properties: - # usually we want to enforce tls, but not for test (greenmail is on localhost w/o tls) - mail.smtp.auth: true - mail.smtp.starttls.enable: true - mail.smtp.starttls.required: true - # https://docs.aws.amazon.com/ses/latest/dg/smtp-connect.html STARTTLS port = - # To set up a STARTTLS connection, the SMTP client connects to the Amazon SES SMTP endpoint on port 25, 587, or 2587, - # issues an EHLO (SMTP extended hello) command, and waits for the server to announce that it supports the - # STARTTLS SMTP extension. CAUTION: do not use ports 465 or 2465 here, they are for the TLS Wrapper connection - mail.smtp.port: 2587 - ## various timeouts - mail.smtp.timeout: 5000 - mail.smtp.connectiontimeout: 5000 - mail.smtp.writetimeout: 5000 - # default from address - mail.smtp.from: angkor-thom@localhost + --- -# overwrites for spring 'prod' profile, none-sensitive test specific values only +# spring 'prod' profile overwrites, none-sensitive test specific values only # for sensitive values see config/application.properties, which is NOT under version control # alternatively, sensitive prod values may be injected via environment spring: config: activate: on-profile: prod + flyway: clean-disabled: true jackson: @@ -221,9 +244,10 @@ server: error: include-message: never # disable for prod to avoid leaking of info to client (default for spring >= 2.3) + --- -# overwrites for spring 'test' profile, none-sensitive test specific values only -# for sensitive values see src/test/resources/application-test.properties, which is NOT under version control +# spring 'test' profile overwrites, use for none-sensitive test specific values only +# for sensitive values, see src/test/resources/application-test.properties, which is NOT under version control spring: config: activate: