diff --git a/charts/radar-output/templates/deployment.yaml b/charts/radar-output/templates/deployment.yaml index 94bbed5b8..e373041a1 100644 --- a/charts/radar-output/templates/deployment.yaml +++ b/charts/radar-output/templates/deployment.yaml @@ -36,7 +36,7 @@ spec: - key: "app.kubernetes.io/name" operator: In values: - - {{ template "app-config-frontend.name" . }} + - {{ template "radar-output.name" . }} - key: "app.kubernetes.io/instance" operator: In values: diff --git a/charts/radar-s3-connector/README.md b/charts/radar-s3-connector/README.md index 14b9203c4..23902005d 100644 --- a/charts/radar-s3-connector/README.md +++ b/charts/radar-s3-connector/README.md @@ -46,7 +46,8 @@ A Helm chart for RADAR-base s3 connector. This connector uses Confluent s3 conne | tolerations | list | `[]` | Toleration labels for pod assignment | | affinity | object | `{}` | Affinity labels for pod assignment | | kafka.url | string | `"PLAINTEXT://cp-kafka-headless:9092"` | Kafka broker URLs | -| schemaRegistry.url | string | `"http://cp-schema-registry:8081"` | | +| schemaRegistry.url | string | `"http://cp-schema-registry:8081"` | Schema registry URL | +| catalogServer.url | string | `"http://catalog-server:9010"` | Catalog server URL | | topics | string | check values.yaml | List of topics to be consumed by the sink connector separated by comma. | | s3Endpoint | string | `"http://minio:9000/"` | Target S3 endpoint url | | s3Tagging | bool | `false` | set to true, if S3 objects should be tagged with start and end offsets, as well as record count. | diff --git a/charts/radar-s3-connector/templates/configmap.yaml b/charts/radar-s3-connector/templates/configmap.yaml index 9b87620a8..d9b545eb6 100644 --- a/charts/radar-s3-connector/templates/configmap.yaml +++ b/charts/radar-s3-connector/templates/configmap.yaml @@ -46,3 +46,11 @@ data: elif [ $(echo "$STATUS" | grep RUNNING | wc -l) -le 1 ]; then exit 1 fi + add-topics-to-config.sh: | + #!/bin/sh + SOURCE=$1 + TARGET=$2 + set -e + apk add --no-cache curl jq + TOPICS=$(curl -f {{ .Values.catalogServer.url }}/source-types | jq -r '[to_entries[] | select(.key|endswith("source-types")) | .value[].data[].topic] | unique | join(",")') + sed "s/^topics=.\+\$/\\0,$TOPICS/" "$SOURCE" | sed "s/^topics=\$/\\0$TOPICS/" > "$TARGET" diff --git a/charts/radar-s3-connector/templates/deployment.yaml b/charts/radar-s3-connector/templates/deployment.yaml index 48946c668..378519eb1 100644 --- a/charts/radar-s3-connector/templates/deployment.yaml +++ b/charts/radar-s3-connector/templates/deployment.yaml @@ -204,7 +204,7 @@ spec: exec: command: - /bin/sh - - /etc/kafka-connect/sink-s3/healthcheck.sh + - /etc/kafka-connect/sink-s3/original/healthcheck.sh initialDelaySeconds: 40 periodSeconds: 60 timeoutSeconds: 10 @@ -214,7 +214,7 @@ spec: exec: command: - /bin/sh - - /etc/kafka-connect/sink-s3/healthcheck.sh + - /etc/kafka-connect/sink-s3/original/healthcheck.sh initialDelaySeconds: 40 periodSeconds: 60 timeoutSeconds: 10 @@ -223,12 +223,29 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: - - name: config + - name: config-original + mountPath: /etc/kafka-connect/sink-s3/original + - name: config-updated + mountPath: /etc/kafka-connect/sink-s3 + initContainers: + - name: init-topics + image: alpine:latest + args: + - /bin/sh + - /etc/kafka-connect/sink-s3/original/add-topics-to-config.sh + - /etc/kafka-connect/sink-s3/original/sink-s3.properties + - /etc/kafka-connect/sink-s3/sink-s3.properties + volumeMounts: + - name: config-original + mountPath: /etc/kafka-connect/sink-s3/original + - name: config-updated mountPath: /etc/kafka-connect/sink-s3 volumes: - - name: config + - name: config-original configMap: name: {{ include "radar-s3-connector.fullname" . }} + - name: config-updated + emptyDir: {} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/radar-s3-connector/values.yaml b/charts/radar-s3-connector/values.yaml index 92701ad31..2ea226609 100644 --- a/charts/radar-s3-connector/values.yaml +++ b/charts/radar-s3-connector/values.yaml @@ -69,9 +69,13 @@ kafka: url: PLAINTEXT://cp-kafka-headless:9092 schemaRegistry: - # Schema registry URL + # -- Schema registry URL url: http://cp-schema-registry:8081 +catalogServer: + # -- Catalog server URL + url: http://catalog-server:9010 + # -- List of topics to be consumed by the sink connector separated by comma. # @default -- check values.yaml topics: android_phone_usage_event_output,android_biovotion_vsm1_acceleration,android_biovotion_vsm1_battery_level,android_biovotion_vsm1_blood_volume_pulse,android_biovotion_vsm1_energy,android_biovotion_vsm1_galvanic_skin_response,android_biovotion_vsm1_heartrate,android_biovotion_vsm1_heartrate_variability,android_biovotion_vsm1_led_current,android_biovotion_vsm1_oxygen_saturation,android_biovotion_vsm1_ppg_raw,android_biovotion_vsm1_respiration_rate,android_biovotion_vsm1_temperature,android_bittium_faros_acceleration,android_bittium_faros_battery_level,android_bittium_faros_ecg,android_bittium_faros_inter_beat_interval,android_bittium_faros_temperature,android_empatica_e4_acceleration,android_empatica_e4_battery_level,android_empatica_e4_blood_volume_pulse,android_empatica_e4_electrodermal_activity,android_empatica_e4_inter_beat_interval,android_empatica_e4_sensor_status,android_empatica_e4_temperature,android_local_weather,android_pebble_2_acceleration,android_pebble_2_battery_level,android_pebble_2_heartrate,android_pebble_2_heartrate_filtered,android_phone_acceleration,android_phone_battery_level,android_phone_bluetooth_devices,android_phone_call,android_phone_contacts,android_phone_gyroscope,android_phone_light,android_phone_magnetic_field,android_phone_ppg,android_phone_relative_location,android_phone_sms,android_phone_sms_unread,android_phone_step_count,android_phone_usage_event,android_phone_user_interaction,android_processed_audio,application_device_info,application_external_time,application_record_counts,application_server_status,application_time_zone,application_uptime,certh_banking_app_event,certh_banking_app_transaction,connect_fitbit_activity_log,connect_fitbit_intraday_calories,connect_fitbit_intraday_heart_rate,connect_fitbit_intraday_steps,connect_fitbit_sleep_classic,connect_fitbit_sleep_stages,connect_fitbit_time_zone,connect_upload_altoida_acceleration,connect_upload_altoida_action,connect_upload_altoida_attitude,connect_upload_altoida_bit_metrics,connect_upload_altoida_blink,connect_upload_altoida_diagnostics,connect_upload_altoida_domain_result,connect_upload_altoida_dot_metrics,connect_upload_altoida_eye_tracking,connect_upload_altoida_gravity,connect_upload_altoida_magnetic_field,connect_upload_altoida_metadata,connect_upload_altoida_object,connect_upload_altoida_path,connect_upload_altoida_rotation,connect_upload_altoida_summary,connect_upload_altoida_tap,connect_upload_altoida_touch,connect_upload_axivity_acceleration,connect_upload_axivity_battery_level,connect_upload_axivity_event,connect_upload_axivity_light,connect_upload_axivity_metadata,connect_upload_axivity_temperature,connect_upload_oxford_camera_data,connect_upload_oxford_camera_image,connect_upload_physilog_binary_data,notification_thinc_it,questionnaire_app_event,questionnaire_ari_self,questionnaire_art_cognitive_test,questionnaire_audio,questionnaire_baars_iv,questionnaire_bipq,questionnaire_completion_log,questionnaire_esm,questionnaire_esm28q,questionnaire_esm_epi_mod_1,questionnaire_evening_assessment,questionnaire_gad7,questionnaire_morning_assessment,questionnaire_patient_determined_disease_step,questionnaire_perceived_deficits_questionnaire,questionnaire_phq8,questionnaire_rpq,questionnaire_rses,questionnaire_tam,questionnaire_timezone,task_2MW_test,task_romberg_test,task_tandem_walking_test,thincit_code_breaker,thincit_pdq5,thincit_spotter,thincit_symbol_check,thincit_trails,