Merge pull request #131 from RADAR-base/automatic-s3-topics

Automatic s3 topics

charts/radar-output/templates/deployment.yaml

@@ -36,7 +36,7 @@ spec:
                   - key: "app.kubernetes.io/name"
                     operator: In
                     values:
-                    - {{ template "app-config-frontend.name" . }}
+                    - {{ template "radar-output.name" . }}
                   - key: "app.kubernetes.io/instance"
                     operator: In
                     values:

charts/radar-s3-connector/README.md

@@ -46,7 +46,8 @@ A Helm chart for RADAR-base s3 connector. This connector uses Confluent s3 conne
 | tolerations | list | `[]` | Toleration labels for pod assignment |
 | affinity | object | `{}` | Affinity labels for pod assignment |
 | kafka.url | string | `"PLAINTEXT://cp-kafka-headless:9092"` | Kafka broker URLs |
-| schemaRegistry.url | string | `"http://cp-schema-registry:8081"` |  |
+| schemaRegistry.url | string | `"http://cp-schema-registry:8081"` | Schema registry URL |
+| catalogServer.url | string | `"http://catalog-server:9010"` | Catalog server URL |
 | topics | string | check values.yaml | List of topics to be consumed by the sink connector separated by comma. |
 | s3Endpoint | string | `"http://minio:9000/"` | Target S3 endpoint url |
 | s3Tagging | bool | `false` | set to true, if S3 objects should be tagged with start and end offsets, as well as record count. |

charts/radar-s3-connector/templates/configmap.yaml

@@ -46,3 +46,11 @@ data:
     elif [ $(echo "$STATUS" | grep RUNNING | wc -l) -le 1 ]; then
       exit 1
     fi
+  add-topics-to-config.sh: |
+    #!/bin/sh
+    SOURCE=$1
+    TARGET=$2
+    set -e
+    apk add --no-cache curl jq
+    TOPICS=$(curl -f {{ .Values.catalogServer.url }}/source-types | jq -r '[to_entries[] | select(.key|endswith("source-types")) | .value[].data[].topic] | unique | join(",")')
+    sed "s/^topics=.\+\$/\\0,$TOPICS/" "$SOURCE" | sed "s/^topics=\$/\\0$TOPICS/" > "$TARGET"

charts/radar-s3-connector/templates/deployment.yaml

@@ -204,7 +204,7 @@ spec:
             exec:
               command:
               - /bin/sh
-              - /etc/kafka-connect/sink-s3/healthcheck.sh
+              - /etc/kafka-connect/sink-s3/original/healthcheck.sh
             initialDelaySeconds: 40
             periodSeconds: 60
             timeoutSeconds: 10
@@ -214,7 +214,7 @@ spec:
             exec:
               command:
               - /bin/sh
-              - /etc/kafka-connect/sink-s3/healthcheck.sh
+              - /etc/kafka-connect/sink-s3/original/healthcheck.sh
             initialDelaySeconds: 40
             periodSeconds: 60
             timeoutSeconds: 10
@@ -223,12 +223,29 @@ spec:
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:
-            - name: config
+            - name: config-original
+              mountPath: /etc/kafka-connect/sink-s3/original
+            - name: config-updated
+              mountPath: /etc/kafka-connect/sink-s3
+      initContainers:
+        - name: init-topics
+          image: alpine:latest
+          args:
+            - /bin/sh
+            - /etc/kafka-connect/sink-s3/original/add-topics-to-config.sh
+            - /etc/kafka-connect/sink-s3/original/sink-s3.properties
+            - /etc/kafka-connect/sink-s3/sink-s3.properties
+          volumeMounts:
+            - name: config-original
+              mountPath: /etc/kafka-connect/sink-s3/original
+            - name: config-updated
               mountPath: /etc/kafka-connect/sink-s3
       volumes:
-        - name: config
+        - name: config-original
           configMap:
             name: {{ include "radar-s3-connector.fullname" . }}
+        - name: config-updated
+          emptyDir: {}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

charts/radar-s3-connector/values.yaml

@@ -69,9 +69,13 @@ kafka:
   url: PLAINTEXT://cp-kafka-headless:9092
 
 schemaRegistry:
-  # Schema registry URL
+  # -- Schema registry URL
   url: http://cp-schema-registry:8081
 
+catalogServer:
+  # -- Catalog server URL
+  url: http://catalog-server:9010
+
 # -- List of topics to be consumed by the sink connector separated by comma.
 # @default -- check values.yaml
 topics: android_phone_usage_event_output,android_biovotion_vsm1_acceleration,android_biovotion_vsm1_battery_level,android_biovotion_vsm1_blood_volume_pulse,android_biovotion_vsm1_energy,android_biovotion_vsm1_galvanic_skin_response,android_biovotion_vsm1_heartrate,android_biovotion_vsm1_heartrate_variability,android_biovotion_vsm1_led_current,android_biovotion_vsm1_oxygen_saturation,android_biovotion_vsm1_ppg_raw,android_biovotion_vsm1_respiration_rate,android_biovotion_vsm1_temperature,android_bittium_faros_acceleration,android_bittium_faros_battery_level,android_bittium_faros_ecg,android_bittium_faros_inter_beat_interval,android_bittium_faros_temperature,android_empatica_e4_acceleration,android_empatica_e4_battery_level,android_empatica_e4_blood_volume_pulse,android_empatica_e4_electrodermal_activity,android_empatica_e4_inter_beat_interval,android_empatica_e4_sensor_status,android_empatica_e4_temperature,android_local_weather,android_pebble_2_acceleration,android_pebble_2_battery_level,android_pebble_2_heartrate,android_pebble_2_heartrate_filtered,android_phone_acceleration,android_phone_battery_level,android_phone_bluetooth_devices,android_phone_call,android_phone_contacts,android_phone_gyroscope,android_phone_light,android_phone_magnetic_field,android_phone_ppg,android_phone_relative_location,android_phone_sms,android_phone_sms_unread,android_phone_step_count,android_phone_usage_event,android_phone_user_interaction,android_processed_audio,application_device_info,application_external_time,application_record_counts,application_server_status,application_time_zone,application_uptime,certh_banking_app_event,certh_banking_app_transaction,connect_fitbit_activity_log,connect_fitbit_intraday_calories,connect_fitbit_intraday_heart_rate,connect_fitbit_intraday_steps,connect_fitbit_sleep_classic,connect_fitbit_sleep_stages,connect_fitbit_time_zone,connect_upload_altoida_acceleration,connect_upload_altoida_action,connect_upload_altoida_attitude,connect_upload_altoida_bit_metrics,connect_upload_altoida_blink,connect_upload_altoida_diagnostics,connect_upload_altoida_domain_result,connect_upload_altoida_dot_metrics,connect_upload_altoida_eye_tracking,connect_upload_altoida_gravity,connect_upload_altoida_magnetic_field,connect_upload_altoida_metadata,connect_upload_altoida_object,connect_upload_altoida_path,connect_upload_altoida_rotation,connect_upload_altoida_summary,connect_upload_altoida_tap,connect_upload_altoida_touch,connect_upload_axivity_acceleration,connect_upload_axivity_battery_level,connect_upload_axivity_event,connect_upload_axivity_light,connect_upload_axivity_metadata,connect_upload_axivity_temperature,connect_upload_oxford_camera_data,connect_upload_oxford_camera_image,connect_upload_physilog_binary_data,notification_thinc_it,questionnaire_app_event,questionnaire_ari_self,questionnaire_art_cognitive_test,questionnaire_audio,questionnaire_baars_iv,questionnaire_bipq,questionnaire_completion_log,questionnaire_esm,questionnaire_esm28q,questionnaire_esm_epi_mod_1,questionnaire_evening_assessment,questionnaire_gad7,questionnaire_morning_assessment,questionnaire_patient_determined_disease_step,questionnaire_perceived_deficits_questionnaire,questionnaire_phq8,questionnaire_rpq,questionnaire_rses,questionnaire_tam,questionnaire_timezone,task_2MW_test,task_romberg_test,task_tandem_walking_test,thincit_code_breaker,thincit_pdq5,thincit_spotter,thincit_symbol_check,thincit_trails,