diff --git a/README.md b/README.md index a4cc030ba..ea96df0c3 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ The Kubernetes stack of RADAR-base platform. * [Configure](#configure) * [Install](#install) - [Usage and accessing the applications](#usage-and-accessing-the-applications) +- [Service-specific documentation](#service-specific-documentation) - [Troubleshooting](#troubleshooting) - [Upgrade instructions](#upgrade-instructions) * [Upgrade to RADAR-Kubernetes version 1.1.x](#upgrade-to-radar-kubernetes-version-11x) @@ -328,6 +329,11 @@ https://k8s.radar-base.org Now you can head over to the [Management Portal](https://radar-base.atlassian.net/wiki/spaces/RAD/pages/49512484/Management+Portal) guide for next steps. +## Service-specific documentation + +- [Data Dashboard Backend data transformation](docs/ksql-server_for_data-dashboard-backend.md) + + ## Troubleshooting If an application doesn't become fully ready, installation will not be successful. In this case, you should investigate the root cause by investigating the relevant component. It's suggested to run the following command when `helmfile sync` command is running so you can keep an eye on the installation: @@ -386,6 +392,15 @@ Run the following instructions to upgrade an existing RADAR-Kubernetes cluster. | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Upgrading the major version of a PostgreSQL image is not supported. If necessary, we propose to use a `pg_dump` to dump the current data and a `pg_restore` to restore that data on a newer version. Please find instructions for this elsewhere. | +### Upgrade to RADAR-Kubernetes version >=1.1.4 + +In `production.yaml` rename sections: + +| Old Name | New Name | +|--------------------------|------------------------------------| +| radar_jdbc_connector | radar_jdbc_connector_grafana | +| radar_jdbc_connector_agg | radar_jdbc_connector_realtime_dashboard | + ### Upgrade to RADAR-Kubernetes version 1.1.x Before running the upgrade make sure to copy `environments.yaml.tmpl` to `environments.yaml` and if you've previously changed `environments.yaml` apply the changes again. This is necessary due to addition of `helmDefaults` and `repositories` configurations to this file. diff --git a/docs/ksql-server_for_data-dashboard-backend.md b/docs/ksql-server_for_data-dashboard-backend.md new file mode 100644 index 000000000..6a377517d --- /dev/null +++ b/docs/ksql-server_for_data-dashboard-backend.md @@ -0,0 +1,46 @@ +# Kafka-data transformer (ksql-server) for data-dashboard-backend service + +Reference: https://docs.ksql-server.io/ + +The data-dashboard-backend service uses data derived from Kafka topics imported into the _observation_ table +data-dashboard-backend service database. The data in the Kafka topics is transformed by the ksql-server data transformer +to be imported into the _observation_ table. + +The ksql-server data transformer is able to register Consumer/Producers to Kafka that transform data in a topic and +publish the results to another topic. + +The provided ksql-server _questionnaire_response_observations.sql_ and _questionnaire_app_events_observation.sql_ SQL files +transform, respectively, the _questionnaire_response_ and _questionnaire_app_event_ topics and publish the data to the +_ksql_observations_ topic. The _ksql_observations_ topic is consumed by the radar-jdbc-connector service deployed for the +data-dashboard-backend service (see: [20-data-dashboard.yaml](../helmfile.d/20-dashboard.yaml)). + +When transformation of other topics is required, new SQL files can be added to this directory. These new files should be +referenced in the _cp-ksql-server_ -> ksql -> queries_ section of the `etc/base.yaml.gotmpl` file. New ksql-server SQL +files should transform towards the following format of the _ksql_observations_ topic: + +``` + TOPIC KEY: + PROJECT: the project identifier + SOURCE: the source identifier + SUBJECT: the subject/study participant identifier + TOPIC VALUE: + TOPIC: the topic identifier + CATEGORY: the category identifier (optional) + VARIABLE: the variable identifier + DATE: the date of the observation + END_DATE: the end date of the observation (optional) + TYPE: the type of the observation (STRING, STRING_JSON, INTEGER, DOUBLE) + VALUE_TEXTUAL: the textual value of the observation (optional, must be set when VALUE_NUMERIC is NULL) + VALUE_NUMERIC: the numeric value of the observation (optional, must be set when VALUE_TEXTUAL is NULL) +``` + +New messages are added to the _ksql_observations_ topic by inserting into the _observations_ stream ( +see [_base_observations_stream.sql](_base_observations_stream.sql)): + +``` +INSERT INTO observations +SELECT +... +PARTITION BY q.projectId, q.userId, q.sourceId +EMIT CHANGES; +``` \ No newline at end of file diff --git a/etc/base-secrets.yaml b/etc/base-secrets.yaml index 39af09cbb..47241cc2a 100644 --- a/etc/base-secrets.yaml +++ b/etc/base-secrets.yaml @@ -90,6 +90,8 @@ management_portal: client_secret: secret radar_push_endpoint: client_secret: secret + radar_data_dashboard_backend: + client_secret: secret smtp: password: change_me @@ -124,8 +126,9 @@ oura_api_secret: change_me radar_rest_sources_backend: postgres: password: secret -# --------------------------------------------------------- 20-grafana.yaml --------------------------------------------------------- +# --------------------------------------------------------- 20-dashboard.yaml --------------------------------------------------------- timescaledb_password: secret +data_dashboard_backend_db_password: secret grafana_password: secret grafana_metrics_password: secret diff --git a/etc/base.yaml b/etc/base.yaml index 2bb093ce9..c9e6e577f 100644 --- a/etc/base.yaml +++ b/etc/base.yaml @@ -290,10 +290,12 @@ radar_rest_sources_backend: garmin: enable: "false" -# --------------------------------------------------------- 20-grafana.yaml --------------------------------------------------------- +# --------------------------------------------------------- 20-dashboard.yaml --------------------------------------------------------- timescaledb_username: postgres timescaledb_db_name: grafana-metrics +data_dashboard_backend_db_name: data-dashboard +data_dashboard_backend_db_username: postgres grafana_metrics_username: postgres timescaledb: @@ -315,16 +317,29 @@ timescaledb: # Uncomment when upgrading #existingClaim: "data-timescaledb-postgresql-0" +# Make sure to set: +#- radar_jdbc_connector_grafana._install to true +#- ksql_server._install to true radar_grafana: - _install: true + _install: false _chart_version: 6.26.8 _extra_timeout: 0 replicaCount: 1 env: GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/allprojects/home.json -radar_jdbc_connector: - _install: true +# Make sure to set: +#- radar_jdbc_connector_data_dashboard_backend._install to true +#- ksql_server._install to true +data_dashboard_backend: + _install: false + _chart_version: 0.1.0 + _extra_timeout: 0 + replicaCount: 1 + +# Install when radar_grafana._install is 'true' +radar_jdbc_connector_grafana: + _install: false _chart_version: 0.5.1 _extra_timeout: 0 replicaCount: 1 @@ -332,6 +347,33 @@ radar_jdbc_connector: # Change the list of topics if you have dashboards that read other data or if you don't have certain topics available on your cluster. topics: android_phone_relative_location, android_phone_battery_level, connect_fitbit_intraday_heart_rate, connect_fitbit_intraday_steps +# Install when data_dashboard_backend._install is 'true' +radar_jdbc_connector_data_dashboard_backend: + _install: false + _chart_version: 0.5.1 + _extra_timeout: 0 + replicaCount: 1 + +# Install when using realtime analysis +radar_jdbc_connector_realtime_dashboard: + _install: false + _chart_version: 0.5.1 + _extra_timeout: 0 + replicaCount: 1 + +# Install when: +#- radar_grafana._install is 'true' +#- data_dashboard_backend._install is 'true' +#- using realtime analysis +ksql_server: + _install: false + _chart_version: 0.3.1 + _extra_timeout: 0 + # -- Uncomment when using real-time analysis + # ksql: + # headless: false + # -- + # --------------------------------------------------------- 20-ingestion.yaml --------------------------------------------------------- radar_gateway: @@ -488,30 +530,6 @@ radar_push_endpoint: garmin: enabled: true -# --------------------------------------------------------- 40-realtime-analyses.yaml --------------------------------------------------------- - -radar_jdbc_connector_agg: - _install: false - _chart_version: 0.5.1 - _extra_timeout: 0 - replicaCount: 1 - -ksql_server: - _install: false - _chart_version: 0.3.1 - _extra_timeout: 0 - replicaCount: 1 - servicePort: 8088 - kafka: - bootstrapServers: PLAINTEXT://cp-kafka:9092 - cp-schema-registry: - url: http://cp-schema-registry:8081 - ksql: - headless: false - configurationOverrides: - "ksql.server.url": "http://0.0.0.0:8088" - "ksql.advertised.listener": "http://ksql-server:8088" - # --------------------------------------------------------- 99-velero.yaml --------------------------------------------------------- velero: diff --git a/etc/base.yaml.gotmpl b/etc/base.yaml.gotmpl index 1eb6439ec..b50d49ebe 100644 --- a/etc/base.yaml.gotmpl +++ b/etc/base.yaml.gotmpl @@ -27,9 +27,15 @@ radar_grafana: # google_application_credentials: {{ readFile "../etc/radar-appserver/firebase-adminsdk.json" | quote }} #*/}} -# Remove below Go comment to read the queries.sql and set the queries -# in the ksql_server -#ksql_server: +# If data transformation of kafka topic data is needed, please remove the Go template comments and yaml comments. +# Make sure to reference a ksql transformation file that contains the required transformation logic. +# The files below are transform the data from the questionnaire_response and questionnaire_app_events topics to the +# ksql_observations topic, used by the data-dashboard-backend. If using the data-dashboard-backend, please make sure +# to uncomment the relevant ksql transformer files. +# Note: never remove the _base_observations_stream.sql file. +# ksql_server: # ksql: # queries: | -# {{/*- readFile "cp-ksql-server/queries.sql" | nindent 8 */}} +# {{/*- readFile "../etc/cp-ksql-server/_base_observations_stream.sql" | nindent 8 */}} +# {{/*- readFile "../etc/cp-ksql-server/questionnaire_response_observations.sql" | nindent 8 */}} +# {{/*- readFile "../etc/cp-ksql-server/questionnaire_app_event_observations.sql" | nindent 8 */}} diff --git a/etc/cp-ksql-server/_base_observations_stream.sql b/etc/cp-ksql-server/_base_observations_stream.sql new file mode 100644 index 000000000..9d44979e2 --- /dev/null +++ b/etc/cp-ksql-server/_base_observations_stream.sql @@ -0,0 +1,20 @@ +SET 'auto.offset.reset' = 'earliest'; + +-- Register the 'ksql_observations' topic (is created when not exists). +CREATE STREAM observations ( + PROJECT VARCHAR KEY, -- 'KEY' means that this field is part of the kafka message key + SUBJECT VARCHAR KEY, + SOURCE VARCHAR KEY, + TOPIC_NAME VARCHAR, + CATEGORY VARCHAR, + VARIABLE VARCHAR, + OBSERVATION_TIME TIMESTAMP, + OBSERVATION_TIME_END TIMESTAMP, + TYPE VARCHAR, + VALUE_NUMERIC DOUBLE, + VALUE_TEXTUAL VARCHAR +) WITH ( + kafka_topic = 'ksql_observations', + partitions = 3, + format = 'avro' +); diff --git a/etc/cp-ksql-server/queries.sql b/etc/cp-ksql-server/queries.sql deleted file mode 100644 index e69de29bb..000000000 diff --git a/etc/cp-ksql-server/questionnaire_app_event_observations.sql b/etc/cp-ksql-server/questionnaire_app_event_observations.sql new file mode 100644 index 000000000..001229036 --- /dev/null +++ b/etc/cp-ksql-server/questionnaire_app_event_observations.sql @@ -0,0 +1,31 @@ +CREATE STREAM questionnaire_app_event ( + projectId VARCHAR KEY, -- 'KEY' means that this field is part of the kafka message key + userId VARCHAR KEY, + sourceId VARCHAR KEY, + questionnaireName VARCHAR, + eventType VARCHAR, + time DOUBLE, + metadata MAP+ +) WITH ( + kafka_topic = 'questionnaire_app_event', + partitions = 3, + format = 'avro' +); + +INSERT INTO observations +WITH (QUERY_ID='questionnaire_app_event_observations') +SELECT + q.projectId AS PROJECT, + q.userId AS SUBJECT, + q.sourceId AS SOURCE, + 'questionnaire_app_event' as TOPIC_NAME, + CAST(NULL as VARCHAR) as CATEGORY, + q.questionnaireName as VARIABLE, + FROM_UNIXTIME(CAST(q.time * 1000 AS BIGINT)) as OBSERVATION_TIME, + CAST(NULL as TIMESTAMP) as OBSERVATION_TIME_END, + 'STRING_JSON' as TYPE, + CAST(NULL as DOUBLE) as VALUE_NUMERIC, + TO_JSON_STRING(q.metadata) as VALUE_TEXTUAL +FROM questionnaire_app_event q +PARTITION BY q.projectId, q.userId, q.sourceId -- this sets the fields in the kafka message key +EMIT CHANGES; diff --git a/etc/cp-ksql-server/questionnaire_response_observations.sql b/etc/cp-ksql-server/questionnaire_response_observations.sql new file mode 100644 index 000000000..69a2c87ae --- /dev/null +++ b/etc/cp-ksql-server/questionnaire_response_observations.sql @@ -0,0 +1,83 @@ +CREATE STREAM questionnaire_response ( + projectId VARCHAR KEY, -- 'KEY' means that this field is part of the kafka message key + userId VARCHAR KEY, + sourceId VARCHAR KEY, + time DOUBLE, + timeCompleted DOUBLE, + timeNotification DOUBLE, + name VARCHAR, + version VARCHAR, + answers ARRAY, startTime DOUBLE, endTime DOUBLE>> +) WITH ( + kafka_topic = 'questionnaire_response', + partitions = 3, + format = 'avro' +); + +CREATE STREAM questionnaire_response_exploded +AS SELECT + EXPLODE(TRANSFORM(q.answers, a => a->questionId)) as VARIABLE, + FROM_UNIXTIME(CAST(q.time * 1000 AS BIGINT)) as OBSERVATION_TIME, + q.projectId, + q.userId, + q.sourceId, + 'questionnaire_response' as TOPIC_NAME, + q.name as CATEGORY, + CAST(NULL as TIMESTAMP) as OBSERVATION_TIME_END, + -- WARNING!!! The cast from VARCHAR (string) to DOUBLE will throw an JAVA exception if the string is not a number. + -- This does not mean that the message will be lost. The value will be present in the VALUE_TEXTUAL_OPTIONAL field. + EXPLODE(TRANSFORM(q.answers, a => COALESCE(a->value->double, CAST(a->value->int as DOUBLE), CAST(a->value->string as DOUBLE)))) as VALUE_NUMERIC, + EXPLODE(TRANSFORM(q.answers, a => CASE + WHEN a->value->int IS NOT NULL THEN 'INTEGER' + WHEN a->value->double IS NOT NULL THEN 'DOUBLE' + ELSE NULL + END)) as TYPE, + -- Note: When cast to double works for the string value, the VALUE_TEXTUAL_OPTIONAL will also be set. + EXPLODE(TRANSFORM(q.answers, a => a->value->string)) as VALUE_TEXTUAL_OPTIONAL +FROM questionnaire_response q +EMIT CHANGES; + +INSERT INTO observations +WITH (QUERY_ID='questionnaire_response_observations') +SELECT + q.projectId as PROJECT, + q.sourceId as SOURCE, + q.userId as SUBJECT, + TOPIC_NAME, CATEGORY, VARIABLE, OBSERVATION_TIME, OBSERVATION_TIME_END, + CASE + WHEN TYPE IS NULL AND VALUE_NUMERIC IS NOT NULL THEN 'DOUBLE' -- must have been derived from a string cast + WHEN TYPE IS NULL AND VALUE_NUMERIC IS NULL THEN 'STRING' + ELSE TYPE -- keep the original type when TYPE is not NULL + END as TYPE, + VALUE_NUMERIC, + CASE + WHEN VALUE_NUMERIC IS NOT NULL THEN NULL -- When cast to double has worked for the string value, set VALUE_TEXTUAL to NULL. + ELSE VALUE_TEXTUAL_OPTIONAL + END as VALUE_TEXTUAL +FROM questionnaire_response_exploded q +PARTITION BY q.projectId, q.userId, q.sourceId -- this sets the fields in the kafka message key +EMIT CHANGES; + +-- TODO: exploding the 'select:' questions is not yet fully designed. +-- I keep the code here for future reference. +-- Multi-select questionnaire questions are stored as a single 'value' string with the +-- names of the selected options separated by comma's. Multiselect questions are prefixed +-- by 'select:' in the questionId. +-- When 'questionId' is like 'select:%' create a new stream with the select options. +-- The options in the value field split commas and added as separate VARIABLE records. +-- The VALUE_NUMERIC is set to 1 and VALUE_TEXTUAL is set to NULL. +-- INSERT INTO observations +-- SELECT +-- EXPLODE(SPLIT(VALUE_TEXTUAL, ',')) as VARIABLE, +-- PROJECT, SOURCE, SUBJECT, TOPIC_NAME, CATEGORY, OBSERVATION_TIME, OBSERVATION_TIME_END, +-- 'INTEGER' as TYPE, +-- CAST(1 as DOUBLE) VALUE_NUMERIC, +-- CAST(NULL as VARCHAR) as VALUE_TEXTUAL +-- FROM questionnaire_response_observations +-- WHERE +-- VARIABLE IS NOT NULL +-- AND VARIABLE LIKE 'select:%' +-- AND VALUE_TEXTUAL IS NOT NULL +-- AND VALUE_TEXTUAL != '' +-- PARTITION BY SUBJECT, PROJECT, SOURCE +-- EMIT CHANGES; diff --git a/etc/cp-ksql-server/values.yaml b/etc/cp-ksql-server/values.yaml index 3e6dafb1e..50b413d11 100644 --- a/etc/cp-ksql-server/values.yaml +++ b/etc/cp-ksql-server/values.yaml @@ -1,6 +1,12 @@ # KSQL configuration options ## ref: https://docs.confluent.io/current/ksql/docs/installation/server-config/config-reference.html +kafka: + bootstrapServers: PLAINTEXT://cp-kafka:9092 +cp-schema-registry: + url: http://cp-schema-registry:8081 configurationOverrides: + "ksql.server.url": "http://0.0.0.0:8088" + "ksql.advertised.listener": "http://ksql-server:8088" "ksql.logging.processing.topic.auto.create": true "ksql.logging.processing.stream.auto.create": true "ksql.service.id": radar_default_ diff --git a/etc/radar-jdbc-connector-data-dashboard-backend/values.yaml b/etc/radar-jdbc-connector-data-dashboard-backend/values.yaml new file mode 100644 index 000000000..4e6968628 --- /dev/null +++ b/etc/radar-jdbc-connector-data-dashboard-backend/values.yaml @@ -0,0 +1,13 @@ +sink: + # Central topic for transformed topics (see docs/KSQLDB_transformer_for_data-dashboard-backend.md) + topics: ksql_observations + tableNameFormat: observation + insertMode: insert + primaryKeys: + fields: + - PROJECT + - SUBJECT + - SOURCE + - TOPIC_NAME + - VARIABLE + - OBSERVATION_TIME \ No newline at end of file diff --git a/etc/radar-jdbc-connector-agg/values.yaml b/etc/radar-jdbc-connector-realtime-dashboard/values.yaml similarity index 100% rename from etc/radar-jdbc-connector-agg/values.yaml rename to etc/radar-jdbc-connector-realtime-dashboard/values.yaml diff --git a/helmfile.d/20-dashboard.yaml b/helmfile.d/20-dashboard.yaml new file mode 100644 index 000000000..000ee0f13 --- /dev/null +++ b/helmfile.d/20-dashboard.yaml @@ -0,0 +1,151 @@ +bases: +- ../environments.yaml + +--- + +{{ readFile "../etc/helmfile-templates.yaml" }} + +releases: + - name: timescaledb + chart: radar/postgresql + version: {{ .Values.timescaledb._chart_version }} + installed: {{ .Values.timescaledb._install }} + timeout: {{ add .Values.base_timeout .Values.timescaledb._extra_timeout }} + <<: *logFailedRelease + values: + - "../etc/timescaledb/values.yaml" + - {{ .Values.timescaledb | toYaml | indent 8 | trim }} + set: + {{- if eq .Values.timescaledb_username "postgres" }} + - name: global.postgresql.auth.postgresPassword + value: {{ .Values.timescaledb_password }} + {{- else }} + - name: global.postgresql.auth.username + value: {{ .Values.timescaledb_username }} + - name: global.postgresql.auth.password + value: {{ .Values.timescaledb_password }} + {{- end }} + - name: auth.database + value: {{ .Values.timescaledb_db_name }} + + - name: radar-grafana + chart: radar/grafana + version: {{ .Values.radar_grafana._chart_version }} + installed: {{ .Values.radar_grafana._install }} + timeout: {{ add .Values.base_timeout .Values.radar_grafana._extra_timeout }} + <<: *logFailedRelease + values: + - "../etc/radar-grafana/values.yaml" + - {{ .Values.radar_grafana | toYaml | indent 8 | trim }} + set: + - name: adminPassword + value: {{ .Values.grafana_password }} + - name: env.POSTGRES_DB + value: {{ .Values.timescaledb_db_name | quote }} + - name: envRenderSecret.POSTGRES_PASSWORD + value: {{ .Values.timescaledb_password }} + - name: ingress.hosts + values: ["dashboard.{{ .Values.server_name }}"] + - name: "grafana\\.ini.server.root_url" + value: "https://dashboard.{{ .Values.server_name }}/" + - name: ingress.tls[0].secretName + value: radar-base-tls-dashboard + - name: ingress.tls[0].hosts + values: ["dashboard.{{ .Values.server_name }}"] + - name: "grafana\\.ini.metrics.basic_auth_username" + value: {{ .Values.grafana_metrics_username }} + - name: "grafana\\.ini.metrics.basic_auth_password" + value: {{ .Values.grafana_metrics_password }} + - name: "grafana\\.ini.auth\\.generic_oauth.auth_url" + value: "https://{{ .Values.server_name }}/managementportal/oauth/authorize" + - name: "grafana\\.ini.auth\\.generic_oauth.token_url" + value: "https://{{ .Values.server_name }}/managementportal/oauth/token" + + - name: data-dashboard-backend + chart: radar/data-dashboard-backend + version: {{ .Values.data_dashboard_backend._chart_version }} + installed: {{ .Values.data_dashboard_backend._install }} + timeout: {{ add .Values.base_timeout .Values.data_dashboard_backend._extra_timeout }} + <<: *logFailedRelease + values: + - {{ .Values.data_dashboard_backend | toYaml | indent 8 | trim }} + set: + - name: serverName + value: {{ .Values.server_name }} + - name: ingress.hosts + values: + - {{ .Values.server_name }} + - name: jdbc.user + value: {{ .Values.timescaledb_username }} + - name: jdbc.password + value: {{ .Values.timescaledb_password }} + - name: jdbc.url + value: {{ dig "jdbc" "url" (printf "jdbc:postgresql://timescaledb-postgresql-hl:5432/%s" .Values.data_dashboard_backend_db_name) .Values.data_dashboard_backend }} + - name: managementPortal.clientSecret + value: {{ .Values.management_portal.oauth_clients.radar_data_dashboard_backend.client_secret }} + + - name: ksql-server + chart: cp-radar/cp-ksql-server + installed: {{ .Values.ksql_server._install }} + version: {{ .Values.ksql_server._chart_version }} + timeout: {{ add .Values.base_timeout .Values.ksql_server._extra_timeout }} + <<: *logFailedRelease + values: + - "../etc/cp-ksql-server/values.yaml" + - {{ .Values.ksql_server | toYaml | indent 8 | trim }} + + - name: radar-jdbc-connector-grafana + chart: radar/radar-jdbc-connector + version: {{ .Values.radar_jdbc_connector_grafana._chart_version }} + installed: {{ .Values.radar_jdbc_connector_grafana._install }} + timeout: {{ add .Values.base_timeout .Values.radar_jdbc_connector_grafana._extra_timeout }} + <<: *logFailedRelease + values: + - {{ .Values.radar_jdbc_connector_grafana | toYaml | indent 8 | trim }} + set: + - name: kafka_num_brokers + value: {{ .Values.kafka_num_brokers }} + - name: jdbc.user + value: {{ .Values.timescaledb_username }} + - name: jdbc.password + value: {{ .Values.timescaledb_password }} + - name: jdbc.url + value: {{ dig "jdbc" "url" (printf "jdbc:postgresql://timescaledb-postgresql-hl:5432/%s" .Values.timescaledb_db_name) .Values.radar_jdbc_connector_realtime_dashboard }} + + - name: radar-jdbc-connector-data-dashboard-backend + chart: radar/radar-jdbc-connector + version: {{ .Values.radar_jdbc_connector_data_dashboard_backend._chart_version }} + installed: {{ .Values.radar_jdbc_connector_data_dashboard_backend._install }} + timeout: {{ add .Values.base_timeout .Values.radar_jdbc_connector_data_dashboard_backend._extra_timeout }} + <<: *logFailedRelease + values: + - "../etc/radar-jdbc-connector-data-dashboard-backend/values.yaml" + - {{ .Values.radar_jdbc_connector_data_dashboard_backend | toYaml | indent 8 | trim }} + set: + - name: kafka_num_brokers + value: {{ .Values.kafka_num_brokers }} + - name: jdbc.user + value: {{ .Values.data_dashboard_backend_db_username }} + - name: jdbc.password + value: {{ .Values.data_dashboard_backend_db_password }} + - name: jdbc.url + value: {{ dig "jdbc" "url" (printf "jdbc:postgresql://timescaledb-postgresql-hl:5432/%s" .Values.data_dashboard_backend_db_name) .Values.radar_jdbc_connector_data_dashboard_backend }} + + - name: radar-jdbc-connector-realtime-dashboard + chart: radar/radar-jdbc-connector + version: {{ .Values.radar_jdbc_connector_realtime_dashboard._chart_version }} + installed: {{ .Values.radar_jdbc_connector_realtime_dashboard._install }} + timeout: {{ add .Values.base_timeout .Values.radar_jdbc_connector_realtime_dashboard._extra_timeout }} + <<: *logFailedRelease + values: + - "../etc/radar-jdbc-connector-realtime-dashboard/values.yaml" + - {{ .Values.radar_jdbc_connector_realtime_dashboard | toYaml | indent 8 | trim }} + set: + - name: kafka_num_brokers + value: {{ .Values.kafka_num_brokers }} + - name: jdbc.user + value: {{ .Values.timescaledb_username }} + - name: jdbc.password + value: {{ .Values.timescaledb_password }} + - name: jdbc.url + value: {{ dig "jdbc" "url" (printf "jdbc:postgresql://timescaledb-postgresql-hl:5432/%s" .Values.timescaledb_db_name) .Values.radar_jdbc_connector_realtime_dashboard }} diff --git a/helmfile.d/20-grafana.yaml b/helmfile.d/20-grafana.yaml deleted file mode 100644 index abc59601b..000000000 --- a/helmfile.d/20-grafana.yaml +++ /dev/null @@ -1,80 +0,0 @@ -bases: -- ../environments.yaml - ---- - -{{ readFile "../etc/helmfile-templates.yaml" }} - -releases: - - name: timescaledb - chart: radar/postgresql - version: {{ .Values.timescaledb._chart_version }} - installed: {{ .Values.timescaledb._install }} - timeout: {{ add .Values.base_timeout .Values.timescaledb._extra_timeout }} - <<: *logFailedRelease - values: - - "../etc/timescaledb/values.yaml" - - {{ .Values.timescaledb | toYaml | indent 8 | trim }} - set: - {{- if eq .Values.timescaledb_username "postgres" }} - - name: global.postgresql.auth.postgresPassword - value: {{ .Values.timescaledb_password }} - {{- else }} - - name: global.postgresql.auth.username - value: {{ .Values.timescaledb_username }} - - name: global.postgresql.auth.password - value: {{ .Values.timescaledb_password }} - {{- end }} - - name: auth.database - value: {{ .Values.timescaledb_db_name }} - - - name: radar-grafana - chart: radar/grafana - version: {{ .Values.radar_grafana._chart_version }} - installed: {{ .Values.radar_grafana._install }} - timeout: {{ add .Values.base_timeout .Values.radar_grafana._extra_timeout }} - <<: *logFailedRelease - values: - - "../etc/radar-grafana/values.yaml" - - {{ .Values.radar_grafana | toYaml | indent 8 | trim }} - set: - - name: adminPassword - value: {{ .Values.grafana_password }} - - name: env.POSTGRES_DB - value: {{ .Values.timescaledb_db_name | quote }} - - name: envRenderSecret.POSTGRES_PASSWORD - value: {{ .Values.timescaledb_password }} - - name: ingress.hosts - values: ["dashboard.{{ .Values.server_name }}"] - - name: "grafana\\.ini.server.root_url" - value: "https://dashboard.{{ .Values.server_name }}/" - - name: ingress.tls[0].secretName - value: radar-base-tls-dashboard - - name: ingress.tls[0].hosts - values: ["dashboard.{{ .Values.server_name }}"] - - name: "grafana\\.ini.metrics.basic_auth_username" - value: {{ .Values.grafana_metrics_username }} - - name: "grafana\\.ini.metrics.basic_auth_password" - value: {{ .Values.grafana_metrics_password }} - - name: "grafana\\.ini.auth\\.generic_oauth.auth_url" - value: "https://{{ .Values.server_name }}/managementportal/oauth/authorize" - - name: "grafana\\.ini.auth\\.generic_oauth.token_url" - value: "https://{{ .Values.server_name }}/managementportal/oauth/token" - - - name: radar-jdbc-connector - chart: radar/radar-jdbc-connector - version: {{ .Values.radar_jdbc_connector._chart_version }} - installed: {{ .Values.radar_jdbc_connector._install }} - timeout: {{ add .Values.base_timeout .Values.radar_jdbc_connector._extra_timeout }} - <<: *logFailedRelease - values: - - {{ .Values.radar_jdbc_connector | toYaml | indent 8 | trim }} - set: - - name: kafka_num_brokers - value: {{ .Values.kafka_num_brokers }} - - name: jdbc.user - value: {{ .Values.timescaledb_username }} - - name: jdbc.password - value: {{ .Values.timescaledb_password }} - - name: jdbc.url - value: {{ dig "jdbc" "url" (printf "jdbc:postgresql://timescaledb-postgresql-hl:5432/%s" .Values.timescaledb_db_name) .Values.radar_jdbc_connector }} diff --git a/helmfile.d/40-realtime-dashboard.yaml b/helmfile.d/40-realtime-dashboard.yaml deleted file mode 100644 index 10f1d1b60..000000000 --- a/helmfile.d/40-realtime-dashboard.yaml +++ /dev/null @@ -1,36 +0,0 @@ -bases: -- ../environments.yaml - ---- - -{{ readFile "../etc/helmfile-templates.yaml" }} - -releases: - - name: radar-jdbc-connector-agg - chart: radar/radar-jdbc-connector - version: {{ .Values.radar_jdbc_connector_agg._chart_version }} - installed: {{ .Values.radar_jdbc_connector_agg._install }} - timeout: {{ add .Values.base_timeout .Values.radar_jdbc_connector_agg._extra_timeout }} - <<: *logFailedRelease - values: - - "../etc/radar-jdbc-connector-agg/values.yaml" - - {{ .Values.radar_jdbc_connector_agg | toYaml | indent 8 | trim }} - set: - - name: kafka_num_brokers - value: {{ .Values.kafka_num_brokers }} - - name: jdbc.user - value: {{ .Values.timescaledb_username }} - - name: jdbc.password - value: {{ .Values.timescaledb_password }} - - name: jdbc.url - value: {{ dig "jdbc" "url" (printf "jdbc:postgresql://timescaledb-postgresql-hl:5432/%s" .Values.timescaledb_db_name) .Values.radar_jdbc_connector_agg }} - - - name: ksql-server - chart: cp-radar/cp-ksql-server - installed: {{ .Values.ksql_server._install }} - version: {{ .Values.ksql_server._chart_version }} - timeout: {{ add .Values.base_timeout .Values.ksql_server._extra_timeout }} - <<: *logFailedRelease - values: - - "../etc/cp-ksql-server/values.yaml" - - {{ .Values.ksql_server | toYaml | indent 8 | trim }}