From 547d1d3bc7e0b9aeb3c0d61740f04a8f2717780a Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 24 Aug 2021 17:08:02 +0200 Subject: [PATCH 1/6] Automatically use right topics in s3 connector --- .../templates/configmap-init.yaml | 18 +++++++++++++++ .../templates/deployment.yaml | 23 +++++++++++++++++-- charts/radar-s3-connector/values.yaml | 4 ++++ 3 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 charts/radar-s3-connector/templates/configmap-init.yaml diff --git a/charts/radar-s3-connector/templates/configmap-init.yaml b/charts/radar-s3-connector/templates/configmap-init.yaml new file mode 100644 index 000000000..111599a81 --- /dev/null +++ b/charts/radar-s3-connector/templates/configmap-init.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "radar-s3-connector.fullname" . }}-init + labels: + app: {{ template "radar-s3-connector.name" . }} + chart: {{ template "radar-s3-connector.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + add-topics-to-config.sh: | + #!/bin/sh + SOURCE=$1 + TARGET=$2 + set -e + apk add --no-cache curl jq + TOPICS=$(curl -f {{ .Values.catalogServer.url }}/source-types | jq -r '[to_entries[] | select(.key|endswith("source-types")) | .value[].data[].topic] | unique | join(",")') + sed "s/^topics=.\+\$/\\0,$TOPICS/" "$SOURCE" | sed "s/^topics=\$/\\0$TOPICS/" > "$TARGET" diff --git a/charts/radar-s3-connector/templates/deployment.yaml b/charts/radar-s3-connector/templates/deployment.yaml index 48946c668..0ebb4ea6d 100644 --- a/charts/radar-s3-connector/templates/deployment.yaml +++ b/charts/radar-s3-connector/templates/deployment.yaml @@ -204,7 +204,7 @@ spec: exec: command: - /bin/sh - - /etc/kafka-connect/sink-s3/healthcheck.sh + - /etc/kafka-connect/sink-s3/original/healthcheck.sh initialDelaySeconds: 40 periodSeconds: 60 timeoutSeconds: 10 @@ -214,7 +214,7 @@ spec: exec: command: - /bin/sh - - /etc/kafka-connect/sink-s3/healthcheck.sh + - /etc/kafka-connect/sink-s3/original/healthcheck.sh initialDelaySeconds: 40 periodSeconds: 60 timeoutSeconds: 10 @@ -222,13 +222,32 @@ spec: failureThreshold: 3 resources: {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: config-updated + mountPath: /etc/kafka-connect/sink-s3 + - name: config + mountPath: /etc/kafka-connect/sink-s3/original + initContainers: + - name: init-topics + image: alpine:latest + args: + - /etc/kafka-connect/sink-s3/init/init-topics.sh /etc/kafka-connect/sink-s3/original/sink-s3.properties /etc/kafka-connect/sink-s3/sink-s3.properties volumeMounts: - name: config + mountPath: /etc/kafka-connect/sink-s3/original + - name: config-init + mountPath: /etc/kafka-connect/sink-s3/init + - name: config-updated mountPath: /etc/kafka-connect/sink-s3 volumes: - name: config configMap: name: {{ include "radar-s3-connector.fullname" . }} + - name: config-init + configMap: + name: {{ include "radar-s3-connector.fullname" . }}-init + - name: config-updated + emptyDir: {} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/radar-s3-connector/values.yaml b/charts/radar-s3-connector/values.yaml index 92701ad31..cfadb89fa 100644 --- a/charts/radar-s3-connector/values.yaml +++ b/charts/radar-s3-connector/values.yaml @@ -72,6 +72,10 @@ schemaRegistry: # Schema registry URL url: http://cp-schema-registry:8081 +catalogServer: + # -- Catalog server URL + url: http://catalog-server:9010 + # -- List of topics to be consumed by the sink connector separated by comma. # @default -- check values.yaml topics: android_phone_usage_event_output,android_biovotion_vsm1_acceleration,android_biovotion_vsm1_battery_level,android_biovotion_vsm1_blood_volume_pulse,android_biovotion_vsm1_energy,android_biovotion_vsm1_galvanic_skin_response,android_biovotion_vsm1_heartrate,android_biovotion_vsm1_heartrate_variability,android_biovotion_vsm1_led_current,android_biovotion_vsm1_oxygen_saturation,android_biovotion_vsm1_ppg_raw,android_biovotion_vsm1_respiration_rate,android_biovotion_vsm1_temperature,android_bittium_faros_acceleration,android_bittium_faros_battery_level,android_bittium_faros_ecg,android_bittium_faros_inter_beat_interval,android_bittium_faros_temperature,android_empatica_e4_acceleration,android_empatica_e4_battery_level,android_empatica_e4_blood_volume_pulse,android_empatica_e4_electrodermal_activity,android_empatica_e4_inter_beat_interval,android_empatica_e4_sensor_status,android_empatica_e4_temperature,android_local_weather,android_pebble_2_acceleration,android_pebble_2_battery_level,android_pebble_2_heartrate,android_pebble_2_heartrate_filtered,android_phone_acceleration,android_phone_battery_level,android_phone_bluetooth_devices,android_phone_call,android_phone_contacts,android_phone_gyroscope,android_phone_light,android_phone_magnetic_field,android_phone_ppg,android_phone_relative_location,android_phone_sms,android_phone_sms_unread,android_phone_step_count,android_phone_usage_event,android_phone_user_interaction,android_processed_audio,application_device_info,application_external_time,application_record_counts,application_server_status,application_time_zone,application_uptime,certh_banking_app_event,certh_banking_app_transaction,connect_fitbit_activity_log,connect_fitbit_intraday_calories,connect_fitbit_intraday_heart_rate,connect_fitbit_intraday_steps,connect_fitbit_sleep_classic,connect_fitbit_sleep_stages,connect_fitbit_time_zone,connect_upload_altoida_acceleration,connect_upload_altoida_action,connect_upload_altoida_attitude,connect_upload_altoida_bit_metrics,connect_upload_altoida_blink,connect_upload_altoida_diagnostics,connect_upload_altoida_domain_result,connect_upload_altoida_dot_metrics,connect_upload_altoida_eye_tracking,connect_upload_altoida_gravity,connect_upload_altoida_magnetic_field,connect_upload_altoida_metadata,connect_upload_altoida_object,connect_upload_altoida_path,connect_upload_altoida_rotation,connect_upload_altoida_summary,connect_upload_altoida_tap,connect_upload_altoida_touch,connect_upload_axivity_acceleration,connect_upload_axivity_battery_level,connect_upload_axivity_event,connect_upload_axivity_light,connect_upload_axivity_metadata,connect_upload_axivity_temperature,connect_upload_oxford_camera_data,connect_upload_oxford_camera_image,connect_upload_physilog_binary_data,notification_thinc_it,questionnaire_app_event,questionnaire_ari_self,questionnaire_art_cognitive_test,questionnaire_audio,questionnaire_baars_iv,questionnaire_bipq,questionnaire_completion_log,questionnaire_esm,questionnaire_esm28q,questionnaire_esm_epi_mod_1,questionnaire_evening_assessment,questionnaire_gad7,questionnaire_morning_assessment,questionnaire_patient_determined_disease_step,questionnaire_perceived_deficits_questionnaire,questionnaire_phq8,questionnaire_rpq,questionnaire_rses,questionnaire_tam,questionnaire_timezone,task_2MW_test,task_romberg_test,task_tandem_walking_test,thincit_code_breaker,thincit_pdq5,thincit_spotter,thincit_symbol_check,thincit_trails, From fcff3b62ebb6ae7d250817b39f4cb0cc913750a7 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 24 Aug 2021 17:14:07 +0200 Subject: [PATCH 2/6] Small fixes --- charts/radar-output/templates/deployment.yaml | 2 +- charts/radar-s3-connector/templates/deployment.yaml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/charts/radar-output/templates/deployment.yaml b/charts/radar-output/templates/deployment.yaml index 94bbed5b8..e373041a1 100644 --- a/charts/radar-output/templates/deployment.yaml +++ b/charts/radar-output/templates/deployment.yaml @@ -36,7 +36,7 @@ spec: - key: "app.kubernetes.io/name" operator: In values: - - {{ template "app-config-frontend.name" . }} + - {{ template "radar-output.name" . }} - key: "app.kubernetes.io/instance" operator: In values: diff --git a/charts/radar-s3-connector/templates/deployment.yaml b/charts/radar-s3-connector/templates/deployment.yaml index 0ebb4ea6d..bb103140e 100644 --- a/charts/radar-s3-connector/templates/deployment.yaml +++ b/charts/radar-s3-connector/templates/deployment.yaml @@ -231,7 +231,9 @@ spec: - name: init-topics image: alpine:latest args: - - /etc/kafka-connect/sink-s3/init/init-topics.sh /etc/kafka-connect/sink-s3/original/sink-s3.properties /etc/kafka-connect/sink-s3/sink-s3.properties + - /etc/kafka-connect/sink-s3/init/init-topics.sh + - /etc/kafka-connect/sink-s3/original/sink-s3.properties + - /etc/kafka-connect/sink-s3/sink-s3.properties volumeMounts: - name: config mountPath: /etc/kafka-connect/sink-s3/original From 0abb43d35eff3e2e96b373c153b268dc7e54287b Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 24 Aug 2021 17:21:46 +0200 Subject: [PATCH 3/6] Fix s3 script location --- charts/radar-s3-connector/templates/deployment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/radar-s3-connector/templates/deployment.yaml b/charts/radar-s3-connector/templates/deployment.yaml index bb103140e..1e7eabd5b 100644 --- a/charts/radar-s3-connector/templates/deployment.yaml +++ b/charts/radar-s3-connector/templates/deployment.yaml @@ -231,7 +231,8 @@ spec: - name: init-topics image: alpine:latest args: - - /etc/kafka-connect/sink-s3/init/init-topics.sh + - /bin/sh + - /etc/kafka-connect/sink-s3/init/add-topics-to-config.sh - /etc/kafka-connect/sink-s3/original/sink-s3.properties - /etc/kafka-connect/sink-s3/sink-s3.properties volumeMounts: From 05f4f40a86e897e0c9293d6eb73b59e515814adb Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 24 Aug 2021 17:25:33 +0200 Subject: [PATCH 4/6] Update README --- charts/radar-s3-connector/README.md | 3 ++- charts/radar-s3-connector/values.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/radar-s3-connector/README.md b/charts/radar-s3-connector/README.md index 14b9203c4..23902005d 100644 --- a/charts/radar-s3-connector/README.md +++ b/charts/radar-s3-connector/README.md @@ -46,7 +46,8 @@ A Helm chart for RADAR-base s3 connector. This connector uses Confluent s3 conne | tolerations | list | `[]` | Toleration labels for pod assignment | | affinity | object | `{}` | Affinity labels for pod assignment | | kafka.url | string | `"PLAINTEXT://cp-kafka-headless:9092"` | Kafka broker URLs | -| schemaRegistry.url | string | `"http://cp-schema-registry:8081"` | | +| schemaRegistry.url | string | `"http://cp-schema-registry:8081"` | Schema registry URL | +| catalogServer.url | string | `"http://catalog-server:9010"` | Catalog server URL | | topics | string | check values.yaml | List of topics to be consumed by the sink connector separated by comma. | | s3Endpoint | string | `"http://minio:9000/"` | Target S3 endpoint url | | s3Tagging | bool | `false` | set to true, if S3 objects should be tagged with start and end offsets, as well as record count. | diff --git a/charts/radar-s3-connector/values.yaml b/charts/radar-s3-connector/values.yaml index cfadb89fa..2ea226609 100644 --- a/charts/radar-s3-connector/values.yaml +++ b/charts/radar-s3-connector/values.yaml @@ -69,7 +69,7 @@ kafka: url: PLAINTEXT://cp-kafka-headless:9092 schemaRegistry: - # Schema registry URL + # -- Schema registry URL url: http://cp-schema-registry:8081 catalogServer: From f93c0d8ab0f4ae91e434d85dee6cd275ddd4154a Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 24 Aug 2021 17:34:33 +0200 Subject: [PATCH 5/6] Simplify S3 setup --- .../templates/configmap-init.yaml | 18 ------------------ .../templates/configmap.yaml | 8 ++++++++ .../templates/deployment.yaml | 12 +++++------- 3 files changed, 13 insertions(+), 25 deletions(-) delete mode 100644 charts/radar-s3-connector/templates/configmap-init.yaml diff --git a/charts/radar-s3-connector/templates/configmap-init.yaml b/charts/radar-s3-connector/templates/configmap-init.yaml deleted file mode 100644 index 111599a81..000000000 --- a/charts/radar-s3-connector/templates/configmap-init.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "radar-s3-connector.fullname" . }}-init - labels: - app: {{ template "radar-s3-connector.name" . }} - chart: {{ template "radar-s3-connector.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -data: - add-topics-to-config.sh: | - #!/bin/sh - SOURCE=$1 - TARGET=$2 - set -e - apk add --no-cache curl jq - TOPICS=$(curl -f {{ .Values.catalogServer.url }}/source-types | jq -r '[to_entries[] | select(.key|endswith("source-types")) | .value[].data[].topic] | unique | join(",")') - sed "s/^topics=.\+\$/\\0,$TOPICS/" "$SOURCE" | sed "s/^topics=\$/\\0$TOPICS/" > "$TARGET" diff --git a/charts/radar-s3-connector/templates/configmap.yaml b/charts/radar-s3-connector/templates/configmap.yaml index 9b87620a8..d9b545eb6 100644 --- a/charts/radar-s3-connector/templates/configmap.yaml +++ b/charts/radar-s3-connector/templates/configmap.yaml @@ -46,3 +46,11 @@ data: elif [ $(echo "$STATUS" | grep RUNNING | wc -l) -le 1 ]; then exit 1 fi + add-topics-to-config.sh: | + #!/bin/sh + SOURCE=$1 + TARGET=$2 + set -e + apk add --no-cache curl jq + TOPICS=$(curl -f {{ .Values.catalogServer.url }}/source-types | jq -r '[to_entries[] | select(.key|endswith("source-types")) | .value[].data[].topic] | unique | join(",")') + sed "s/^topics=.\+\$/\\0,$TOPICS/" "$SOURCE" | sed "s/^topics=\$/\\0$TOPICS/" > "$TARGET" diff --git a/charts/radar-s3-connector/templates/deployment.yaml b/charts/radar-s3-connector/templates/deployment.yaml index 1e7eabd5b..3eb8c0297 100644 --- a/charts/radar-s3-connector/templates/deployment.yaml +++ b/charts/radar-s3-connector/templates/deployment.yaml @@ -223,27 +223,25 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: + - name: config-original + mountPath: /etc/kafka-connect/sink-s3/original - name: config-updated mountPath: /etc/kafka-connect/sink-s3 - - name: config - mountPath: /etc/kafka-connect/sink-s3/original initContainers: - name: init-topics image: alpine:latest args: - /bin/sh - - /etc/kafka-connect/sink-s3/init/add-topics-to-config.sh + - /etc/kafka-connect/sink-s3/original/add-topics-to-config.sh - /etc/kafka-connect/sink-s3/original/sink-s3.properties - /etc/kafka-connect/sink-s3/sink-s3.properties volumeMounts: - - name: config + - name: config-original mountPath: /etc/kafka-connect/sink-s3/original - - name: config-init - mountPath: /etc/kafka-connect/sink-s3/init - name: config-updated mountPath: /etc/kafka-connect/sink-s3 volumes: - - name: config + - name: config-original configMap: name: {{ include "radar-s3-connector.fullname" . }} - name: config-init From ae90ae1e7acc9af981748a10acb9249dc2e922f7 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 25 Aug 2021 10:33:20 +0200 Subject: [PATCH 6/6] Removed unused configmap --- charts/radar-s3-connector/templates/deployment.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/charts/radar-s3-connector/templates/deployment.yaml b/charts/radar-s3-connector/templates/deployment.yaml index 3eb8c0297..378519eb1 100644 --- a/charts/radar-s3-connector/templates/deployment.yaml +++ b/charts/radar-s3-connector/templates/deployment.yaml @@ -244,9 +244,6 @@ spec: - name: config-original configMap: name: {{ include "radar-s3-connector.fullname" . }} - - name: config-init - configMap: - name: {{ include "radar-s3-connector.fullname" . }}-init - name: config-updated emptyDir: {} {{- with .Values.nodeSelector }}