From 4d4f7f6dd072ffd58e5c92dbaf6add779a378594 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Thu, 30 Jan 2025 10:18:37 +0530 Subject: [PATCH 01/39] Add initial structure for the hub-helmcharts --- examples/kafka-hub/helm/hub/Chart.yaml | 0 examples/kafka-hub/helm/hub/confs/Config.toml | 0 examples/kafka-hub/helm/hub/templates/conf.yaml | 0 examples/kafka-hub/helm/hub/templates/deployment.yaml | 0 examples/kafka-hub/helm/hub/templates/service.yaml | 0 examples/kafka-hub/helm/hub/values.yaml | 0 6 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 examples/kafka-hub/helm/hub/Chart.yaml create mode 100644 examples/kafka-hub/helm/hub/confs/Config.toml create mode 100644 examples/kafka-hub/helm/hub/templates/conf.yaml create mode 100644 examples/kafka-hub/helm/hub/templates/deployment.yaml create mode 100644 examples/kafka-hub/helm/hub/templates/service.yaml create mode 100644 examples/kafka-hub/helm/hub/values.yaml diff --git a/examples/kafka-hub/helm/hub/Chart.yaml b/examples/kafka-hub/helm/hub/Chart.yaml new file mode 100644 index 00000000..e69de29b diff --git a/examples/kafka-hub/helm/hub/confs/Config.toml b/examples/kafka-hub/helm/hub/confs/Config.toml new file mode 100644 index 00000000..e69de29b diff --git a/examples/kafka-hub/helm/hub/templates/conf.yaml b/examples/kafka-hub/helm/hub/templates/conf.yaml new file mode 100644 index 00000000..e69de29b diff --git a/examples/kafka-hub/helm/hub/templates/deployment.yaml b/examples/kafka-hub/helm/hub/templates/deployment.yaml new file mode 100644 index 00000000..e69de29b diff --git a/examples/kafka-hub/helm/hub/templates/service.yaml b/examples/kafka-hub/helm/hub/templates/service.yaml new file mode 100644 index 00000000..e69de29b diff --git a/examples/kafka-hub/helm/hub/values.yaml b/examples/kafka-hub/helm/hub/values.yaml new file mode 100644 index 00000000..e69de29b From 139f2ca34e3b38fa7aa4e7817af36581a36d11c2 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Thu, 30 Jan 2025 10:19:14 +0530 Subject: [PATCH 02/39] Add initial structure for the consolidator-helmcharts --- examples/kafka-hub/helm/consolidator/Chart.yaml | 0 examples/kafka-hub/helm/consolidator/confs/Config.toml | 0 examples/kafka-hub/helm/consolidator/templates/conf.yaml | 0 examples/kafka-hub/helm/consolidator/templates/deployment.yaml | 0 examples/kafka-hub/helm/consolidator/templates/service.yaml | 0 examples/kafka-hub/helm/consolidator/values.yaml | 0 6 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 examples/kafka-hub/helm/consolidator/Chart.yaml create mode 100644 examples/kafka-hub/helm/consolidator/confs/Config.toml create mode 100644 examples/kafka-hub/helm/consolidator/templates/conf.yaml create mode 100644 examples/kafka-hub/helm/consolidator/templates/deployment.yaml create mode 100644 examples/kafka-hub/helm/consolidator/templates/service.yaml create mode 100644 examples/kafka-hub/helm/consolidator/values.yaml diff --git a/examples/kafka-hub/helm/consolidator/Chart.yaml b/examples/kafka-hub/helm/consolidator/Chart.yaml new file mode 100644 index 00000000..e69de29b diff --git a/examples/kafka-hub/helm/consolidator/confs/Config.toml b/examples/kafka-hub/helm/consolidator/confs/Config.toml new file mode 100644 index 00000000..e69de29b diff --git a/examples/kafka-hub/helm/consolidator/templates/conf.yaml b/examples/kafka-hub/helm/consolidator/templates/conf.yaml new file mode 100644 index 00000000..e69de29b diff --git a/examples/kafka-hub/helm/consolidator/templates/deployment.yaml b/examples/kafka-hub/helm/consolidator/templates/deployment.yaml new file mode 100644 index 00000000..e69de29b diff --git a/examples/kafka-hub/helm/consolidator/templates/service.yaml b/examples/kafka-hub/helm/consolidator/templates/service.yaml new file mode 100644 index 00000000..e69de29b diff --git a/examples/kafka-hub/helm/consolidator/values.yaml b/examples/kafka-hub/helm/consolidator/values.yaml new file mode 100644 index 00000000..e69de29b From 68093ae8d6cd0cd354b717ac40fb8d9f42aa4db2 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Thu, 30 Jan 2025 13:00:45 +0530 Subject: [PATCH 03/39] Add initial helm-chart setup for the websubhub --- examples/kafka-hub/helm/hub/Chart.yaml | 16 +++++ examples/kafka-hub/helm/hub/confs/Config.toml | 58 +++++++++++++++++++ .../kafka-hub/helm/hub/templates/conf.yaml | 18 ++++++ .../helm/hub/templates/deployment.yaml | 29 ++++++++++ .../kafka-hub/helm/hub/templates/service.yaml | 22 +++++++ examples/kafka-hub/helm/hub/values.yaml | 44 ++++++++++++++ 6 files changed, 187 insertions(+) diff --git a/examples/kafka-hub/helm/hub/Chart.yaml b/examples/kafka-hub/helm/hub/Chart.yaml index e69de29b..6cd9ba47 100644 --- a/examples/kafka-hub/helm/hub/Chart.yaml +++ b/examples/kafka-hub/helm/hub/Chart.yaml @@ -0,0 +1,16 @@ +# ------------------------------------------------------------------------------------- +# +# Copyright (c) 2022, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# +# This software is the property of WSO2 LLC. and its suppliers, if any. +# Dissemination of any information or reproduction of any material contained +# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. +# You may not alter or remove any copyright or other notice from copies of this content. +# +# -------------------------------------------------------------------------------------- + +apiVersion: v2 +appVersion: 12.0.0 +description: A Helm chart for the deployment of WSO2 Websubhub Hub Deployment +name: hub +version: 0.1.58 diff --git a/examples/kafka-hub/helm/hub/confs/Config.toml b/examples/kafka-hub/helm/hub/confs/Config.toml index e69de29b..d8810478 100644 --- a/examples/kafka-hub/helm/hub/confs/Config.toml +++ b/examples/kafka-hub/helm/hub/confs/Config.toml @@ -0,0 +1,58 @@ +[kafkaHub.config] +# Flag to check whether to enable/disable security +SECURITY_ON = true + +# Server ID is is used to uniquely identify each server +# Each server must have a unique ID +SERVER_ID = {{ .Values.deployment.config.server_id | quote }} + +# IP and Port of the Kafka bootstrap node +KAFKA_BOOTSTRAP_NODE = {{ .Values.deployment.config.kafka.bootstrap_node | quote }} + +# Maximum number of records returned in a single call to consumer-poll +KAFKA_CONSUMER_MAX_POLL_RECORDS = {{ .Values.deployment.config.kafka.max_poll_records }} + +# Kafka topic which is stores websub-events for this server +WEBSUB_EVENTS_TOPIC = "websub-events" + +# Consolidator HTTP endpoint to be used to retrieve current state-snapshot +STATE_SNAPSHOT_ENDPOINT = {{ .Values.deployment.config.state_snapshot_endpoint | quote }} + +# The interval in which Kafka consumers wait for new messages +POLLING_INTERVAL = 10.0 + +# The period in which Kafka close method waits to complete +GRACEFUL_CLOSE_PERIOD = 5.0 + +# The port that is used to start the hub +HUB_PORT = {{ .Values.deployment.config.port }} + +# The period between retry requests +MESSAGE_DELIVERY_RETRY_INTERVAL = 3.0 + +# The maximum retry count +MESSAGE_DELIVERY_COUNT = 3 + +# The message delivery timeout +MESSAGE_DELIVERY_TIMEOUT = 10.0 + +# The HTTP status codes for which the client should retry +MESSAGE_DELIVERY_RETRYABLE_STATUS_CODES = [{{ join ", " .Values.deployment.config.retryable_status_codes }}] + +# The Oauth2 authorization related configurations +[kafkaHub.config.OAUTH2_CONFIG] +issuer = {{ .Values.deployment.config.idp.jwt_issuer | quote }} +audience = {{ .Values.deployment.config.idp.jwt_audience | quote }} +jwksUrl = {{ .Values.deployment.config.idp.jwt_jwks_endpoint | quote }} +trustStore = "/home/ballerina/resources/{{ .Values.deployment.config.idp.truststore_name }}" +trustStorePassword = {{ .Values.deployment.config.idp.truststore_password | quote }} + +# The MTLS configurations related to Kafka connection +[kafkaHub.config.KAFKA_MTLS_CONFIG] +cert.path = "/home/ballerina/resources/brokercerts/{{ .Values.deployment.config.kafka.truststore_name }}" +cert.password = {{ .Values.deployment.config.kafka.truststore_password | quote }} +key.keyStore.path = "/home/ballerina/resources/brokercerts/{{ .Values.deployment.config.kafka.keystore_name }}" +key.keyStore.password = {{ .Values.deployment.config.kafka.keystore_password | quote }} + +[ballerina.log] +level = {{ .Values.deployment.config.logLevel | quote }} diff --git a/examples/kafka-hub/helm/hub/templates/conf.yaml b/examples/kafka-hub/helm/hub/templates/conf.yaml index e69de29b..57dcccef 100644 --- a/examples/kafka-hub/helm/hub/templates/conf.yaml +++ b/examples/kafka-hub/helm/hub/templates/conf.yaml @@ -0,0 +1,18 @@ +# ------------------------------------------------------------------------------------- +# +# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# +# This software is the property of WSO2 LLC. and its suppliers, if any. +# Dissemination of any information or reproduction of any material contained +# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. +# You may not alter or remove any copyright or other notice from copies of this content. +# +# -------------------------------------------------------------------------------------- + +apiVersion: v1 +kind: ConfigMap +metadata: + name: websubhub-svc-cm + namespace : {{ .Release.Namespace }} +data: + config.toml: {{ tpl (.Files.Get "confs/Config.toml") . | quote }} diff --git a/examples/kafka-hub/helm/hub/templates/deployment.yaml b/examples/kafka-hub/helm/hub/templates/deployment.yaml index e69de29b..67d37619 100644 --- a/examples/kafka-hub/helm/hub/templates/deployment.yaml +++ b/examples/kafka-hub/helm/hub/templates/deployment.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-deployment + labels: + app: {{ .Release.Name }} +spec: + replicas: {{ .Values.deployment.replicas }} + selector: + matchLabels: + app: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ .Release.Name }} + spec: + containers: + - name: {{ .Release.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + ports: + - containerPort: {{ .Values.deployment.config.port }} + volumeMounts: + - name: "config-config-map-volume" + mountPath: "/home/ballerina" + readOnly: true + volumes: + - name: "config-config-map-volume" + configMap: + name: "websubhub-svc-cm" diff --git a/examples/kafka-hub/helm/hub/templates/service.yaml b/examples/kafka-hub/helm/hub/templates/service.yaml index e69de29b..ddfb4351 100644 --- a/examples/kafka-hub/helm/hub/templates/service.yaml +++ b/examples/kafka-hub/helm/hub/templates/service.yaml @@ -0,0 +1,22 @@ +# ------------------------------------------------------------------------------------- +# +# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# +# This software is the property of WSO2 LLC. and its suppliers, if any. +# Dissemination of any information or reproduction of any material contained +# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. +# You may not alter or remove any copyright or other notice from copies of this content. +# +# -------------------------------------------------------------------------------------- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-service +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }} + ports: + - protocol: TCP + port: {{ .Values.deployment.config.port }} + targetPort: {{ .Values.deployment.config.port }} diff --git a/examples/kafka-hub/helm/hub/values.yaml b/examples/kafka-hub/helm/hub/values.yaml index e69de29b..50b324f3 100644 --- a/examples/kafka-hub/helm/hub/values.yaml +++ b/examples/kafka-hub/helm/hub/values.yaml @@ -0,0 +1,44 @@ +# ------------------------------------------------------------------------------------- +# +# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# +# This software is the property of WSO2 LLC. and its suppliers, if any. +# Dissemination of any information or reproduction of any material contained +# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. +# You may not alter or remove any copyright or other notice from copies of this content. +# +# -------------------------------------------------------------------------------------- +deployment: + replicas: 1 + image: + repository: "ayeshalmeida/kafkahub" + pullPolicy: IfNotPresent + tag: 12.0.0 + pullPolicy: "IfNotPresent" + config: + port: 9000 + server_id: "hub-1" + state_snapshot_endpoint: "http://consolidator:10001" + retryable_status_codes: [500, 502, 503] + logLevel: "ERROR" + idp: + jwt_issuer: "https://localhost:9443/oauth2/token" + jwt_audience: "ballerina" + jwt_jwks_endpoint: "https://idp:9443/oauth2/jwks" + truststore_name: "client-truststore.jks" + truststore_password: "wso2carbon" + kafka: + bootstrap_node: "broker:9094" + max_poll_records: 50 + truststore_name: "client-truststore.jks" + truststore_password: "password" + keystore_name: "client-keystore.jks" + keystore_password: "password" + +ingress: + hosts: + - host: ballerina-websubhub.local + paths: + - path: /hub + pathType: Prefix + tls: [] From 176534c07d5f996f2c1ca9a64603abc023399470 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Thu, 30 Jan 2025 13:03:35 +0530 Subject: [PATCH 04/39] Fix missing configuration --- examples/kafka-hub/helm/hub/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kafka-hub/helm/hub/templates/deployment.yaml b/examples/kafka-hub/helm/hub/templates/deployment.yaml index 67d37619..7397ef66 100644 --- a/examples/kafka-hub/helm/hub/templates/deployment.yaml +++ b/examples/kafka-hub/helm/hub/templates/deployment.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - name: {{ .Release.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: "{{ .Values.deployment.image.repository }}:{{ .Values.deployment.image.tag }}" ports: - containerPort: {{ .Values.deployment.config.port }} volumeMounts: From 70ee7914cb5997ffc7e5d1cb93a6e1628891139b Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Thu, 30 Jan 2025 17:18:14 +0530 Subject: [PATCH 05/39] Add initial helm-chart setup for the consolidator --- .../kafka-hub/helm/consolidator/Chart.yaml | 16 ++++++++++ .../helm/consolidator/confs/Config.toml | 25 ++++++++++++++++ .../helm/consolidator/templates/conf.yaml | 18 +++++++++++ .../consolidator/templates/deployment.yaml | 29 ++++++++++++++++++ .../helm/consolidator/templates/service.yaml | 23 ++++++++++++++ .../kafka-hub/helm/consolidator/values.yaml | 30 +++++++++++++++++++ 6 files changed, 141 insertions(+) diff --git a/examples/kafka-hub/helm/consolidator/Chart.yaml b/examples/kafka-hub/helm/consolidator/Chart.yaml index e69de29b..ea4ce0c8 100644 --- a/examples/kafka-hub/helm/consolidator/Chart.yaml +++ b/examples/kafka-hub/helm/consolidator/Chart.yaml @@ -0,0 +1,16 @@ +# ------------------------------------------------------------------------------------- +# +# Copyright (c) 2022, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# +# This software is the property of WSO2 LLC. and its suppliers, if any. +# Dissemination of any information or reproduction of any material contained +# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. +# You may not alter or remove any copyright or other notice from copies of this content. +# +# -------------------------------------------------------------------------------------- + +apiVersion: v2 +appVersion: 7.0.0 +description: A Helm chart for the deployment of WSO2 Websubhub Consolidator Deployment +name: consolidator +version: 0.1.24 diff --git a/examples/kafka-hub/helm/consolidator/confs/Config.toml b/examples/kafka-hub/helm/consolidator/confs/Config.toml index e69de29b..44b3eb0e 100644 --- a/examples/kafka-hub/helm/consolidator/confs/Config.toml +++ b/examples/kafka-hub/helm/consolidator/confs/Config.toml @@ -0,0 +1,25 @@ +[consolidatorService.config] +# IP and Port of the Kafka bootstrap node +KAFKA_BOOTSTRAP_NODE = {{ .Values.deployment.config.kafka.bootstrap_node | quote }} + +# Kafka topic which stores websub-events for this server +WEBSUB_EVENTS_TOPIC = {{ .Values.deployment.config.kafka.websub_events_topic | quote }} + +# Kafka topic which stores the current snapshot for the websub-events +WEBSUB_EVENTS_SNAPSHOT_TOPIC = {{ .Values.deployment.config.kafka.websub_events_snapshot_topic | quote }} + +# The interval in which Kafka consumers wait for new messages +POLLING_INTERVAL = 10.0 + +# The period in which Kafka close method waits to complete +GRACEFUL_CLOSE_PERIOD = 5.0 + +# The port that is used to start the HTTP endpoint for consolidator +CONSOLIDATOR_HTTP_ENDPOINT_PORT = {{ .Values.deployment.config.port }} + +# The MTLS configurations related to Kafka connection +[kafkaHub.config.KAFKA_MTLS_CONFIG] +cert.path = "/home/ballerina/resources/brokercerts/{{ .Values.deployment.config.kafka.truststore_name }}" +cert.password = {{ .Values.deployment.config.kafka.truststore_password | quote }} +key.keyStore.path = "/home/ballerina/resources/brokercerts/{{ .Values.deployment.config.kafka.keystore_name }}" +key.keyStore.password = {{ .Values.deployment.config.kafka.keystore_password | quote }} diff --git a/examples/kafka-hub/helm/consolidator/templates/conf.yaml b/examples/kafka-hub/helm/consolidator/templates/conf.yaml index e69de29b..1019af6b 100644 --- a/examples/kafka-hub/helm/consolidator/templates/conf.yaml +++ b/examples/kafka-hub/helm/consolidator/templates/conf.yaml @@ -0,0 +1,18 @@ +# ------------------------------------------------------------------------------------- +# +# Copyright (c) 2022, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# +# This software is the property of WSO2 LLC. and its suppliers, if any. +# Dissemination of any information or reproduction of any material contained +# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. +# You may not alter or remove any copyright or other notice from copies of this content. +# +# -------------------------------------------------------------------------------------- + +apiVersion: v1 +kind: ConfigMap +metadata: + name: consolidator-svc-cm + namespace : {{ .Release.Namespace }} +data: + config.toml: {{ tpl (.Files.Get "confs/Config.toml") . | quote }} diff --git a/examples/kafka-hub/helm/consolidator/templates/deployment.yaml b/examples/kafka-hub/helm/consolidator/templates/deployment.yaml index e69de29b..f7f80ba1 100644 --- a/examples/kafka-hub/helm/consolidator/templates/deployment.yaml +++ b/examples/kafka-hub/helm/consolidator/templates/deployment.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-deployment + labels: + app: {{ .Release.Name }} +spec: + replicas: {{ .Values.deployment.replicas }} + selector: + matchLabels: + app: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ .Release.Name }} + spec: + containers: + - name: {{ .Release.Name }} + image: "{{ .Values.deployment.image.repository }}:{{ .Values.deployment.image.tag }}" + ports: + - containerPort: {{ .Values.deployment.config.port }} + volumeMounts: + - name: "config-config-map-volume" + mountPath: "/home/ballerina" + readOnly: true + volumes: + - name: "config-config-map-volume" + configMap: + name: "consolidator-svc-cm" diff --git a/examples/kafka-hub/helm/consolidator/templates/service.yaml b/examples/kafka-hub/helm/consolidator/templates/service.yaml index e69de29b..e744d21a 100644 --- a/examples/kafka-hub/helm/consolidator/templates/service.yaml +++ b/examples/kafka-hub/helm/consolidator/templates/service.yaml @@ -0,0 +1,23 @@ +# ------------------------------------------------------------------------------------- +# +# Copyright (c) 2022, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# +# This software is the property of WSO2 LLC. and its suppliers, if any. +# Dissemination of any information or reproduction of any material contained +# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. +# You may not alter or remove any copyright or other notice from copies of this content. +# +# -------------------------------------------------------------------------------------- + +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-service +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }} + ports: + - protocol: TCP + port: {{ .Values.deployment.config.port }} + targetPort: {{ .Values.deployment.config.port }} diff --git a/examples/kafka-hub/helm/consolidator/values.yaml b/examples/kafka-hub/helm/consolidator/values.yaml index e69de29b..5209b96f 100644 --- a/examples/kafka-hub/helm/consolidator/values.yaml +++ b/examples/kafka-hub/helm/consolidator/values.yaml @@ -0,0 +1,30 @@ +# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# +# This software is the property of WSO2 LLC. and its suppliers, if any. +# Dissemination of any information or reproduction of any material contained +# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. +# You may not alter or remove any copyright or other notice from copies of this content. +# +# -------------------------------------------------------------------------------------- +deployment: + replicas: 1 + image: + repository: "ayeshalmeida/consolidator" + pullPolicy: IfNotPresent + tag: 7.0.0 + pullPolicy: "IfNotPresent" + config: + port: 10001 + kafka: + bootstrap_node: "broker:9094" + truststore_name: "client-truststore.jks" + truststore_password: "password" + keystore_name: "client-keystore.jks" + keystore_password: "password" + +ingress: + hosts: + - host: ballerina-consolidator.local + paths: + - path: /consolidator + pathType: Prefix From 1cc1145774fa9aac3d60c1bc18854441d6e1c14e Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Thu, 30 Jan 2025 17:20:53 +0530 Subject: [PATCH 06/39] Fix missing configurations --- examples/kafka-hub/helm/consolidator/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/kafka-hub/helm/consolidator/values.yaml b/examples/kafka-hub/helm/consolidator/values.yaml index 5209b96f..36bc61ab 100644 --- a/examples/kafka-hub/helm/consolidator/values.yaml +++ b/examples/kafka-hub/helm/consolidator/values.yaml @@ -21,6 +21,8 @@ deployment: truststore_password: "password" keystore_name: "client-keystore.jks" keystore_password: "password" + websub_events_topic: "websub-events" + websub_events_snapshot_topic: "websub-events-snapshot" ingress: hosts: From 2822b89a11c2f370d166b7c0d91bc7517409bddd Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Thu, 30 Jan 2025 17:25:33 +0530 Subject: [PATCH 07/39] Add WSO2 licensing header to the code --- .../kafka-hub/helm/consolidator/Chart.yaml | 19 +++++++++++------- .../helm/consolidator/confs/Config.toml | 16 +++++++++++++++ .../helm/consolidator/templates/conf.yaml | 19 +++++++++++------- .../consolidator/templates/deployment.yaml | 16 +++++++++++++++ .../helm/consolidator/templates/service.yaml | 19 +++++++++++------- .../kafka-hub/helm/consolidator/values.yaml | 20 +++++++++++++------ examples/kafka-hub/helm/hub/Chart.yaml | 19 +++++++++++------- examples/kafka-hub/helm/hub/confs/Config.toml | 16 +++++++++++++++ .../kafka-hub/helm/hub/templates/conf.yaml | 19 +++++++++++------- .../helm/hub/templates/deployment.yaml | 16 +++++++++++++++ .../kafka-hub/helm/hub/templates/service.yaml | 20 ++++++++++++------- examples/kafka-hub/helm/hub/values.yaml | 20 ++++++++++++------- 12 files changed, 164 insertions(+), 55 deletions(-) diff --git a/examples/kafka-hub/helm/consolidator/Chart.yaml b/examples/kafka-hub/helm/consolidator/Chart.yaml index ea4ce0c8..c393918c 100644 --- a/examples/kafka-hub/helm/consolidator/Chart.yaml +++ b/examples/kafka-hub/helm/consolidator/Chart.yaml @@ -1,13 +1,18 @@ -# ------------------------------------------------------------------------------------- +# Copyright (c) 2025 WSO2 LLC. (http://www.wso2.com). # -# Copyright (c) 2022, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at # -# This software is the property of WSO2 LLC. and its suppliers, if any. -# Dissemination of any information or reproduction of any material contained -# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. -# You may not alter or remove any copyright or other notice from copies of this content. +# http://www.apache.org/licenses/LICENSE-2.0 # -# -------------------------------------------------------------------------------------- +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. apiVersion: v2 appVersion: 7.0.0 diff --git a/examples/kafka-hub/helm/consolidator/confs/Config.toml b/examples/kafka-hub/helm/consolidator/confs/Config.toml index 44b3eb0e..9d3e53ab 100644 --- a/examples/kafka-hub/helm/consolidator/confs/Config.toml +++ b/examples/kafka-hub/helm/consolidator/confs/Config.toml @@ -1,3 +1,19 @@ +# Copyright (c) 2025 WSO2 LLC. (http://www.wso2.com). +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [consolidatorService.config] # IP and Port of the Kafka bootstrap node KAFKA_BOOTSTRAP_NODE = {{ .Values.deployment.config.kafka.bootstrap_node | quote }} diff --git a/examples/kafka-hub/helm/consolidator/templates/conf.yaml b/examples/kafka-hub/helm/consolidator/templates/conf.yaml index 1019af6b..df7f8fc9 100644 --- a/examples/kafka-hub/helm/consolidator/templates/conf.yaml +++ b/examples/kafka-hub/helm/consolidator/templates/conf.yaml @@ -1,13 +1,18 @@ -# ------------------------------------------------------------------------------------- +# Copyright (c) 2025 WSO2 LLC. (http://www.wso2.com). # -# Copyright (c) 2022, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at # -# This software is the property of WSO2 LLC. and its suppliers, if any. -# Dissemination of any information or reproduction of any material contained -# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. -# You may not alter or remove any copyright or other notice from copies of this content. +# http://www.apache.org/licenses/LICENSE-2.0 # -# -------------------------------------------------------------------------------------- +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. apiVersion: v1 kind: ConfigMap diff --git a/examples/kafka-hub/helm/consolidator/templates/deployment.yaml b/examples/kafka-hub/helm/consolidator/templates/deployment.yaml index f7f80ba1..a73be9ee 100644 --- a/examples/kafka-hub/helm/consolidator/templates/deployment.yaml +++ b/examples/kafka-hub/helm/consolidator/templates/deployment.yaml @@ -1,3 +1,19 @@ +# Copyright (c) 2025 WSO2 LLC. (http://www.wso2.com). +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + apiVersion: apps/v1 kind: Deployment metadata: diff --git a/examples/kafka-hub/helm/consolidator/templates/service.yaml b/examples/kafka-hub/helm/consolidator/templates/service.yaml index e744d21a..1cd7e88f 100644 --- a/examples/kafka-hub/helm/consolidator/templates/service.yaml +++ b/examples/kafka-hub/helm/consolidator/templates/service.yaml @@ -1,13 +1,18 @@ -# ------------------------------------------------------------------------------------- +# Copyright (c) 2025 WSO2 LLC. (http://www.wso2.com). # -# Copyright (c) 2022, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at # -# This software is the property of WSO2 LLC. and its suppliers, if any. -# Dissemination of any information or reproduction of any material contained -# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. -# You may not alter or remove any copyright or other notice from copies of this content. +# http://www.apache.org/licenses/LICENSE-2.0 # -# -------------------------------------------------------------------------------------- +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. apiVersion: v1 kind: Service diff --git a/examples/kafka-hub/helm/consolidator/values.yaml b/examples/kafka-hub/helm/consolidator/values.yaml index 36bc61ab..d8299865 100644 --- a/examples/kafka-hub/helm/consolidator/values.yaml +++ b/examples/kafka-hub/helm/consolidator/values.yaml @@ -1,11 +1,19 @@ -# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# Copyright (c) 2025 WSO2 LLC. (http://www.wso2.com). # -# This software is the property of WSO2 LLC. and its suppliers, if any. -# Dissemination of any information or reproduction of any material contained -# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. -# You may not alter or remove any copyright or other notice from copies of this content. +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at # -# -------------------------------------------------------------------------------------- +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + deployment: replicas: 1 image: diff --git a/examples/kafka-hub/helm/hub/Chart.yaml b/examples/kafka-hub/helm/hub/Chart.yaml index 6cd9ba47..f49cba52 100644 --- a/examples/kafka-hub/helm/hub/Chart.yaml +++ b/examples/kafka-hub/helm/hub/Chart.yaml @@ -1,13 +1,18 @@ -# ------------------------------------------------------------------------------------- +# Copyright (c) 2025 WSO2 LLC. (http://www.wso2.com). # -# Copyright (c) 2022, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at # -# This software is the property of WSO2 LLC. and its suppliers, if any. -# Dissemination of any information or reproduction of any material contained -# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. -# You may not alter or remove any copyright or other notice from copies of this content. +# http://www.apache.org/licenses/LICENSE-2.0 # -# -------------------------------------------------------------------------------------- +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. apiVersion: v2 appVersion: 12.0.0 diff --git a/examples/kafka-hub/helm/hub/confs/Config.toml b/examples/kafka-hub/helm/hub/confs/Config.toml index d8810478..03e1847b 100644 --- a/examples/kafka-hub/helm/hub/confs/Config.toml +++ b/examples/kafka-hub/helm/hub/confs/Config.toml @@ -1,3 +1,19 @@ +# Copyright (c) 2025 WSO2 LLC. (http://www.wso2.com). +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [kafkaHub.config] # Flag to check whether to enable/disable security SECURITY_ON = true diff --git a/examples/kafka-hub/helm/hub/templates/conf.yaml b/examples/kafka-hub/helm/hub/templates/conf.yaml index 57dcccef..f45fafcb 100644 --- a/examples/kafka-hub/helm/hub/templates/conf.yaml +++ b/examples/kafka-hub/helm/hub/templates/conf.yaml @@ -1,13 +1,18 @@ -# ------------------------------------------------------------------------------------- +# Copyright (c) 2025 WSO2 LLC. (http://www.wso2.com). # -# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at # -# This software is the property of WSO2 LLC. and its suppliers, if any. -# Dissemination of any information or reproduction of any material contained -# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. -# You may not alter or remove any copyright or other notice from copies of this content. +# http://www.apache.org/licenses/LICENSE-2.0 # -# -------------------------------------------------------------------------------------- +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. apiVersion: v1 kind: ConfigMap diff --git a/examples/kafka-hub/helm/hub/templates/deployment.yaml b/examples/kafka-hub/helm/hub/templates/deployment.yaml index 7397ef66..a712bea8 100644 --- a/examples/kafka-hub/helm/hub/templates/deployment.yaml +++ b/examples/kafka-hub/helm/hub/templates/deployment.yaml @@ -1,3 +1,19 @@ +# Copyright (c) 2025 WSO2 LLC. (http://www.wso2.com). +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + apiVersion: apps/v1 kind: Deployment metadata: diff --git a/examples/kafka-hub/helm/hub/templates/service.yaml b/examples/kafka-hub/helm/hub/templates/service.yaml index ddfb4351..1cd7e88f 100644 --- a/examples/kafka-hub/helm/hub/templates/service.yaml +++ b/examples/kafka-hub/helm/hub/templates/service.yaml @@ -1,13 +1,19 @@ -# ------------------------------------------------------------------------------------- +# Copyright (c) 2025 WSO2 LLC. (http://www.wso2.com). # -# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at # -# This software is the property of WSO2 LLC. and its suppliers, if any. -# Dissemination of any information or reproduction of any material contained -# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. -# You may not alter or remove any copyright or other notice from copies of this content. +# http://www.apache.org/licenses/LICENSE-2.0 # -# -------------------------------------------------------------------------------------- +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + apiVersion: v1 kind: Service metadata: diff --git a/examples/kafka-hub/helm/hub/values.yaml b/examples/kafka-hub/helm/hub/values.yaml index 50b324f3..21beff9a 100644 --- a/examples/kafka-hub/helm/hub/values.yaml +++ b/examples/kafka-hub/helm/hub/values.yaml @@ -1,13 +1,19 @@ -# ------------------------------------------------------------------------------------- +# Copyright (c) 2025 WSO2 LLC. (http://www.wso2.com). # -# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at # -# This software is the property of WSO2 LLC. and its suppliers, if any. -# Dissemination of any information or reproduction of any material contained -# herein in any form is strictly forbidden, unless permitted by WSO2 expressly. -# You may not alter or remove any copyright or other notice from copies of this content. +# http://www.apache.org/licenses/LICENSE-2.0 # -# -------------------------------------------------------------------------------------- +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + deployment: replicas: 1 image: From 801d40ede259e987f5582916d3517d642a6057e1 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Thu, 30 Jan 2025 17:29:12 +0530 Subject: [PATCH 08/39] Fix config issues in the consolidator helm-charts --- examples/kafka-hub/helm/consolidator/templates/deployment.yaml | 3 ++- examples/kafka-hub/helm/consolidator/values.yaml | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/kafka-hub/helm/consolidator/templates/deployment.yaml b/examples/kafka-hub/helm/consolidator/templates/deployment.yaml index a73be9ee..41cb9810 100644 --- a/examples/kafka-hub/helm/consolidator/templates/deployment.yaml +++ b/examples/kafka-hub/helm/consolidator/templates/deployment.yaml @@ -32,7 +32,8 @@ spec: spec: containers: - name: {{ .Release.Name }} - image: "{{ .Values.deployment.image.repository }}:{{ .Values.deployment.image.tag }}" + image: {{ .Values.deployment.image.repository }}:{{ .Values.deployment.image.tag }} + imagePullPolicy: {{ .Values.deployment.image.pullPolicy }} ports: - containerPort: {{ .Values.deployment.config.port }} volumeMounts: diff --git a/examples/kafka-hub/helm/consolidator/values.yaml b/examples/kafka-hub/helm/consolidator/values.yaml index d8299865..c82868a2 100644 --- a/examples/kafka-hub/helm/consolidator/values.yaml +++ b/examples/kafka-hub/helm/consolidator/values.yaml @@ -20,7 +20,6 @@ deployment: repository: "ayeshalmeida/consolidator" pullPolicy: IfNotPresent tag: 7.0.0 - pullPolicy: "IfNotPresent" config: port: 10001 kafka: From 01e0a6a6a2645ea60121f2bf5704ff0f1fb10281 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Thu, 30 Jan 2025 17:30:31 +0530 Subject: [PATCH 09/39] Fix config issues in the websubhub helm-charts --- examples/kafka-hub/helm/hub/templates/deployment.yaml | 3 ++- examples/kafka-hub/helm/hub/values.yaml | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/kafka-hub/helm/hub/templates/deployment.yaml b/examples/kafka-hub/helm/hub/templates/deployment.yaml index a712bea8..7e863d79 100644 --- a/examples/kafka-hub/helm/hub/templates/deployment.yaml +++ b/examples/kafka-hub/helm/hub/templates/deployment.yaml @@ -32,7 +32,8 @@ spec: spec: containers: - name: {{ .Release.Name }} - image: "{{ .Values.deployment.image.repository }}:{{ .Values.deployment.image.tag }}" + image: {{ .Values.deployment.image.repository }}:{{ .Values.deployment.image.tag }} + imagePullPolicy: {{ .Values.deployment.image.pullPolicy }} ports: - containerPort: {{ .Values.deployment.config.port }} volumeMounts: diff --git a/examples/kafka-hub/helm/hub/values.yaml b/examples/kafka-hub/helm/hub/values.yaml index 21beff9a..d4272e42 100644 --- a/examples/kafka-hub/helm/hub/values.yaml +++ b/examples/kafka-hub/helm/hub/values.yaml @@ -20,7 +20,6 @@ deployment: repository: "ayeshalmeida/kafkahub" pullPolicy: IfNotPresent tag: 12.0.0 - pullPolicy: "IfNotPresent" config: port: 9000 server_id: "hub-1" From b039ea28bb933bc6f30fa358484611d1d115d5c5 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 31 Jan 2025 16:39:36 +0530 Subject: [PATCH 10/39] Add support to mount secrets to the pods --- examples/kafka-hub/helm/hub/Chart.yaml | 2 +- .../helm/hub/_resources/hub-1.keystore.jks | Bin 0 -> 2726 bytes .../hub/_resources/idp/client-truststore.jks | Bin 0 -> 1178 bytes .../kafka-client/kafka-client.keystore.jks | Bin 0 -> 2756 bytes .../kafka-client/kafka-client.trustStore.jks | Bin 0 -> 3776 bytes .../helm/hub/templates/deployment.yaml | 12 ++++++++ .../kafka-hub/helm/hub/templates/secret.yaml | 26 ++++++++++++++++++ examples/kafka-hub/helm/hub/values.yaml | 18 ++++++++++++ 8 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 examples/kafka-hub/helm/hub/_resources/hub-1.keystore.jks create mode 100644 examples/kafka-hub/helm/hub/_resources/idp/client-truststore.jks create mode 100644 examples/kafka-hub/helm/hub/_resources/kafka-client/kafka-client.keystore.jks create mode 100644 examples/kafka-hub/helm/hub/_resources/kafka-client/kafka-client.trustStore.jks create mode 100644 examples/kafka-hub/helm/hub/templates/secret.yaml diff --git a/examples/kafka-hub/helm/hub/Chart.yaml b/examples/kafka-hub/helm/hub/Chart.yaml index f49cba52..8d1eb310 100644 --- a/examples/kafka-hub/helm/hub/Chart.yaml +++ b/examples/kafka-hub/helm/hub/Chart.yaml @@ -18,4 +18,4 @@ apiVersion: v2 appVersion: 12.0.0 description: A Helm chart for the deployment of WSO2 Websubhub Hub Deployment name: hub -version: 0.1.58 +version: 0.1.0 diff --git a/examples/kafka-hub/helm/hub/_resources/hub-1.keystore.jks b/examples/kafka-hub/helm/hub/_resources/hub-1.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..81c74401aa2a8d90e1d2dfe941e39db30424cc97 GIT binary patch literal 2726 zcma);X*kpk_s3_qlS#LbC1u}!V;yT4+1DXUw0y5 ziCq^)V%PjFs{tf<*?(B@Y#0fi^;^FGyY3=5{+;4v2Sf5maQ)w+4sh{L0?q}v0O)^4 zPCyREH%X20>DnA(x33gJEn0QC`#E}(ArPq}5U?1)iD3Qzix4(A2oOfFT_Br+@lY68 z7ItLHs=1wtO9~LFNL#eKgm?!d!3Jvq!o%C-)VIecQ0z6CIUHZ#OX-21i5?eMhL?I-Buz1AD!j4%TPZt@sLDGuihWc~iTe{LZvx`s zEKgRwhiI5*sB_YTfSBU;rqLMZT9e74DoVyvmAvLq9$u^Lcl18*6L3$QE5A{R{o%Wa zkp@LmY_Rv@y48NN&0%E`bbGNTW$7k=`1b*S(Yk594^kbsOUwGknq;gfNpXyRQ@m(W2&BQ*P`1r?FdnEzdx2Jd6Z+jho@;g0P+0yMe{QbBC!B zm)n)DJ;O`i6#vz_X&bYB-C+%?OyrF%ZCz$@db)IPa&lR5e)@Q~4>D?V{&}H<0!4^i zG7S-J2w^KJ|CLA}2fHionIF3Foo8jfs6bc82vO4ez=*z_v`*N7-ClsRR`7}(65Qul z)Q0O?pbIXzp1)F%cklb&`*Jje^aDGxEqz@r)z&b|1J|5YmE&izOc%-Uzl~R@mcaML zhUWehs`b>qsC@8*c5rL+)dd~8nUm*w{)M)@N_U9))XT^Bm|F4+37${)nV#}OGCoi- zQImVV9I|a;x5DK6*yO1@`KyoV;%^#bOKvZmFVH}3eAcsGXd%}ny+v;pa~GfiPF~C<7KaPcl8E)zgtfI^7!TmFK7%XT;*f^-BcYzrdK9 z581O^6622hmz#ydWK+-ysHbH>FnE13)^rVc}uf490|dn{X8QJ zOzuIUK=SH2FOs#UPQf`#LxVP2*Q(hir7GJJm$gyovH7NXi_Akq&h2EUZDSsRb8e;a zURV#pa52Q>dmkr26+Mw}zov$nh?sd86I~!GwEH(T6w$Rg$M)%)V*R5J-5RI1 zD>g2sPgy>64prQDLvc25YeRfkhAdY#GE zYq#?&dgAT7V)1H|x*RY5wlA)~UJZU?rPSD()kKK$OTq}$flD{o+Yhn+&s#IbT4m}C z9!Aob83Pr=3UD`-;I0Sn8uhAtboUHY3+Q*^jd&eJdc8_BZurOf%J`X%E8p{K7{C;g z`NtAARBnJ$r={y3jcU*4H<#)U=^>29_E@avSeF!DZFFM=7)0ckzemD9h@YjPYf$~s zg?ky@vo0eSH);c_6hlIu>r%z)=qvp`)gFgCe#*Nq$IS)_!umnRsD6CBh~Nf7M=&o& z@RgvG{W`h(CsKg%f>#ep@!ylAa-QoA8Zy!ACvQR7A1z;m5bU;cU)oh__W-#-`o2{MdRe#%!~ zNt$dG575{QDDRLXB(pjHz#Bd4&~7^Fp&&teRAHS@@}a#^*V^3c(LR2MGc%PMIC9Dc z--o%ZOe66(#Vz00v^KD83w<%Y*|y+e2>R4s-J!a|e0UZU@P)6ZlX2>QXqW)sME27hIL8TBG$7cr@OZ zld4G|dOT2qXSSq5)2FWqnkZ)a+vbG?TmX1>de}$z)pKh)tNCvo`ZxrBu(zIuOV6>I z2#ll}aksz5mvmo9U|Fggy#3>4X2#w99bo>)tF3viI#eu9rRYjpz?-Y3!N+1BZ0Eq! z&P5TA$~E@%Ka`{@FTXlBgVrQV5#BPWc@$;bAs!a7p5`JHn|6)Ieebf} zLXaR4v9+e+RW}0S87h_77EBox9n*~L^=0mF2Lb97d#Ul!)|-kaVSz#M*C+TVI97Ax z>E^}ik?1AKV~xJ?2%|{RPuJ1o05b0p(qZ_uOoMJ@Lp_h~04y65QBDzY>1VQjg^oi8 z8`P`Sh##5BkCWML`57 zxXVD$CGBb>tC2JB=X{lB&deK@eYMREu8~;5WUQx$grfvHqsLl`)yE3egS7PFoLmJC zwr=wnCcC~AvROs#I9WUHVEe{G!cFhY!y%z2k7{%2sTJi&)900*Na!*~Vgqj0n7=x^ zxTG1#?OIBd49`rn85chNIvWJTQU;I52_)GzJMOhDe6@ z4FLxRpn?P?FoFaj0s#Opf&>}{2`Yw2hW8Bt2LUiC1_~;MNQU673<%~BLmVncsZ0s{cUP=JC1;G%I)*r+@Q(q^(clIDF3M1LBxQ4I?53nw)t zz+`ZeKmzFnQoJ`%bM-PwN{e*IAT$3GXC|_T&;rtJ3uOY3E|@pM(j26qD@m}1d@b{ z*ELX%?0IE$=fT&Y)z1S6otSK$C6Y`dnuEP#zxMpK#7PQj*b57__Fyyrp5D+ABLEg| ziGTw+IHi7*MBw;$D1HW7EHZzW)M1Coc3^n328@2IZY3tBTMf;r>6A?-k7cwaygG|B zn{BIPI&F#9D8&t2G(|;W_MKewwvRoXdl!u4o(Ew29NLx$G1%39RQ3LdMxG)NKg!?+ z^BVzN$B1bB_$NuOj_@7xv?sh8DBwcE7jAV6MyXG8F=x2LUR5Xy>&r&&R0LBTHcsT% zSQf?XK4LR}RJy4+I24-$Mh3^twLOd!$N~9onKmpr`I46WDq<*yrWbwh=+gF9>@0IzDEk<3VhT`a9&y|5`ZZZ{8q5w)vOs(dQcp^cINX;iM%Qc zH_^00md4uqbJo>&zIKJJ1gx`1+UuIyNc@vNG71r-VzXg!wL;7A>qdfRHaNVAt>CWJ z%j33sAb*vu<JyW1Y=S~+%sX=!sIk&m|4E=!)&0K)6QQBcprAiE`8JqV zYfiR$rSSy*aNb`t*oND(h8-mB|E#FMWd3u~+q)FCLe_H)%sbmc* z99g;1DO6GEZe>10tLDIGsVyoGVONy&om;pLNDFM9fn*Cf$RP{U7kJ$pk+sp(20L(s zMI~!XwNtChX*xpCifoG+A%9k)3=}f>9-Kjes4zY-AutIB1uG5%0vZJX1QhK6A(3H0 sZi8bT;b7&RzO8cR{OEZLRpTgbleC1jfnvg<1u`Y^JLU6wM&$PzUQ zgX~`vWyu)pAX3*|p1$Y)y7$L9=Q;0ke!kDa;@I0kKsGFn-5ZLKPS8u(;RM2fg*bLQ zFpk~o5H`c&AT9qzK`OyGNZBF$?63-;$p5%F*nyxz9OTL&sDgF>?SOD%ZLmszuN>I3 zU_R&3Dwa{Q^%7`gVBqn!4~P#@2_O(iiW>w(VL71ie_sS4AONfg6ycPh2Xtct1Es;- z`S-f)OA#4=^m8mi#gS zv#dH3VCAidns-tG0_)uv2$=0It8p8Yk~t&S-~skj;VY1sx3dd3w5-%j_t`&|-=bWE z(~H~3mr@HX?Zq`z@!`lC(Kd`cQDHtlahA|&1Eyz4l?VkjBR=pvaGpul`g2z749|9w z--7u0=OHHI^Q4(wf$-K)<8iNS^D{!w6WS|LcoU319c_{(*b8%w&N&*p8n&tM*r}w& zt?MDE3p>$x5W++t3$!v&*>t_IKA&=QsX(L{=+f+?@|30YfRa@B|#c7?)m6A}m9>84sR4x)0-9OurUREP~ zic?v-?1;V*L3MKH99sNbL>eON8hqJe7jZpij7IDJV)BCceJ(lH+z)g7{FdO-g4Kx1 z8aFY%gm~Zgny8Btg3KAL>7{_V&Jq@eVpy@wtl?JtCqRfU*USdJ)~)=y>8%vzTg+Hh zkotOFZ$hO?2D#eD!IN@J^YTumq4{15KyqxQYoZGMyEzG*8UQRd?a zSAK(aE5o>}qHKlZS39|6bbCjZ#0yC8db{~upVCD87@wXUnw~ym$Ru@1dzeC}Q|XiW zwU%53RtSKk-9f|5BcVxwIx4OWTmpl8NvUSvY+^3yp|`E>n16$>O${lms`_YpjC%=7 zTmr7c?43aZi%^Yy&G6jmZuR0eWKQ-Bh3Q+4+SNt-|9%r6mBbPT>y{Nx?(1f0ZG zjKsCCy*K9qIeQM7re)ejZ#0E{_1qXc$q)>uuLd}tIMVU0x?}B>dK-1R61gG>kb+)Vmj@JFR>PHroS#6D~SNpsc=in)nt=5jM+I2@R} z1_PWfzI{S|Il7!z&3G}~wq*U8<-#(q%Y|jaDKv{)IWgk%qPiX)u^yF#x=T;S$=Sx@z<|J>S34#Cw3fJ|LTui;z?tOMPl}_8x3A2DvbD;C$nDK_8anC)va$|7?!9^naBf-4V{NmyCJ6AO z952q#%#V@Y^qXclgeM(G-QCEPsju6}-?MdwpZ7|Mo^9_pzjblp14ik?QzW781ZSkc z8{dg7#eJUi&t>Y|5zQK7YkNaSV8jQKB9$+L-Zc7MIX6(46TZYKzMVrB>JGK{;Y+l3 zHlLW#c6?=S7FZ(TvS0D~%ccEz4 z1~1@yz9aa0k(gv%Xvnb?RBFaSqZE^x0n6ewx9#|L5_d&7@*r?S1)?p_waV@TuU|WM zX~6U~>w}R+rIE<)5?`wFJ=7SzBIGDRTWe9n+tH^cRxBqXE4U!5G{_P4#eDk$>1cR4K3@MeQxs6=EM+!U zZOgQ$M@P?7oKAzLBqtg?p6@rFh1cR5JqV(?eX&Dp7p4k$`+&MJ4`t_0G1@s&R&sHz zs!R)i{*NQeLw{INJ$&(0T8u_6w77sqR_=YH zMSEZEaH&FVdGxO$K$#8j6`ljVFWHLHGXrMnnv~>mj5y+S#w9FqCU?$=O3;&vNL#N6 z)Z!~^SMX{^w&{Ig-#^fE9toUuHV^fO?rgHj6r`49HiFvRmX+KpOYLr)HVR&x;QAWeH% zC~@}TORLp zVxt-fg_Hb(4%Q?;!9DW+QHm1e&(Q|2n?)V+J6z~IFcA7c*6{AEa`5G(t=de=iHqAK ztn@j{2pMk4*|K9JrPew-`2pUJ^R0TS)(c##@kna}Ji4v_TDnsooIN)EaYct9DSO(e zFmnTIjFrW5{{HMhKnMVg3RKjPYx2L)M^^oAaq-1Nw7`?JqAX8C<7s|~J4VfZ8O(QY a&R1B-us@1i>qs(aq3zh8|7-4_5%dpIH|{wA literal 0 HcmV?d00001 diff --git a/examples/kafka-hub/helm/hub/_resources/kafka-client/kafka-client.trustStore.jks b/examples/kafka-hub/helm/hub/_resources/kafka-client/kafka-client.trustStore.jks new file mode 100644 index 0000000000000000000000000000000000000000..9c71e77fa120529ab5dbe2aba7ba831d76a03fbd GIT binary patch literal 3776 zcma)9Ra6uTv>k>4W{4pNq+1$>hD&!NFm!i^3?U#ALrUreK|n%~8d?+_LPDfN=>Z8T zk?xM?i*@gMU+;bFbJp7Doc+1iK5!JY4TysaM?qcjA-wUL@#iEs1UN+~s2Kh^kxX z@?|!D__XP2_IrJXA;@k*odFQYMFzxSffM6{|636V!2`e<@ger{nm8`FARJy0*`x?s zC%0^SV^cH3Org1W0fYj5!x1#R1(u90ejMvH()vUV66H|#U>)0YUUXXNux={cYbK~# z#b)n*24jgTtDKHI*KkdKzN^%j+m=148I^hx<>N9u=cf}NU&pk&y|V(Tg$_5-_O+!k z{P<24dIsx99BEtQZWMB-rt zWl%pOY1a=fjwM2I&&uzHD*U6A&d0>P_eO*5lPvT6RVkMnvIL}Wp6I* z)b*FL#i`swzc0iNp(jOFNtj=l1`x_0651vXH?URBRrahH{GmKaG6$J;?}wp7mqm^LU)#HBv2FUStk z%LGhVyu-hqazfxeyXeTyHDIDkzsUG_Sz^fIGB{z|TXcI`-i>MUGoCZ#kh$Je+`4bI z&=8#Ra#cDQe$W*6sN7bP$Uw$aCy~C4V-?XCyeV`>QPPnj8k_~9>UYT5{##MD*pI5p zi$qF)r4V|T-`vxXbM`RlQbw0!rE#)6M)t|rC-fH*0cp|IE(JgPXPF~6c2(m(SRMTy z8D9y)A^s>v>T;$vkpKgxN|!g%_g*J~TiXOf6tT(2E%F14lp;R(2DR6Uzm^X(q(_UD zW^1L%6(B#5Ig>cl#am)%@0PP~NNqx7f9nTB9wHY@86}=p-y$Y?Vu|WGOgjIwX0P{DAxPrDpvkLZi~f@G9!p2rA2G!x9r(M)l3wr zP`koxL(h)_+0iXFKlNfY>=K!cC+pmd-Q&Ge*cO+JoP~;+SZ2UPav+Pp4acKyM_s7i z^VS7)EE~XlVeIL4^YXz!y&YZha4MWz%8lJR+-L{lYrW|^4EXkU4BmeleUyQ*6K zvnOl;cMs>p2&yK<=fTHd7;vu9veBlPf)qh`TWVqRmdr>O@%3DgYzKocbRKJlBr(f5 zx*PIl)TVdcY^_ozzJZT3v?Qt^3P7|y!Dd}#V9JextKb)X9G%Im2#o3UmHnAjo0HM9 zw=`>r8NHh7JSdF8)-mt<;-1+Was6SZ@VuKl`0QIbNd8b^Fsj7%v(mDC!ZSPO^{+uD zeJd73*K|LpSQHov>q+H|EBEh?47>?tf3LT%Zx>@va3k7jH4@Fp`>A=g}PRf^5-8CA_BaA70r*8`*V!K~BN! zbkk#gmv2vVrs=x1h49`5$K=Eu|2&58&05Oj-Nu!!cQn9u)PF6GL}T5LlX`)x8QRIh z=5Wz}>=HsH{E&$Q;0161xc*la05}1>0UiJsfG;2rAn*rBKoB4R5C~`e_Xs9T1)?za z@Nr=gmV}GJC8eY#MWx|zI0_v7&k_NC5ekg_L-_)60Dqd>zZ0DQ%f7LkPN7nTiBsq@ zNhQHhs9$0lq0Ilsz871QMO@5&E|<$6RX5q3pGTnx4k{}!A6x-e%1HwcL~5r`@vf-X zhqx+dtgr}&#XXp6iF{ToN1^iwkl4#LpJx*3Uzj^oj{v1>r9uf>?0amN4<+=IMGqu% z=H#QLZ|#fjSE3IFkUX-K>H5;kpLIk(HH~I#Hkm~CkW>sT5cu*2XVvmWe`~ycx=x~u zoPva9rm*CfM(xC#+YguMoPHRIHN7R*rW{b{!y=Y<8hMZVoCIxEZOjS_Y>@E+Swu}7(O~qL)FrNiq}fhd z#7=QB#*jXZaW(RXpgPGH8kOr)ekQicNNzx2ul3*LaU}vb)c}~Z;Dx?h#?3=DW0nf` z)D{MxgcsVbT%taCd@)DTfgK91G-0ZB-z12Km>(sl>sYLRU7j{%pG|PwkHaT)mC(zm zdQO4zkh`faSV}!fA+~IHsm@T@X?G00g);#fHfv)Aa;_t<1<=^|RX}svomD+?$@gCmUj{E#l=9)}9q6wU_)yWnW zzJzpn)e|Jl555&47G8~&7~8F`cH_*Ab>bcJK3hJJUs&9<+oW^uJUJ~B5L;mt9$5futdF*_Jega7%^VB3S~&k!q`M^N zr8dJp+krV^xy|iqzNZqi-uYwui>LVK`M}W_K~CM6ILh8z-?9V& zf}OzTg|?^&R#H3IE$elbn_}q~Qy{7(wBEk6X?;UI!R_Xa(@;E)ewbsTfk+U8S%`;7 zc1a2ER|lbepr2QlP?vkLv`NL4t@2pp~QRTl4gsv{|W&<%2CBN>A(b4 z(=L4v+zt9PmXs~x;}Fdn>Zz!0*1PC*CoRcDs!TVNy;!*YHUN!(VUeUp86Ei{Rr3R1i)YPxZaDIAm+M;Usc&P zR{@!*iu}u0FkkG>l=It{WX~h*d{ID{?+lyZ`)aR|OfSw>IIM%71Vs}ymV6d()4_i%KQBE;bJdY|Sj z&4Mu*CktJC2hx+&>6+>NuMs0xcf*f(Dw)du#;K8iN*RBZQ4rOTKEWkuq;;P;K%QT& zCNif3UOqHMoDts!)+k5w^O=NC^Aj+JEfEP0^*sfgFjj<&$oE(ss*@x0f>2!rVJsg$ew{z%KF7TK#D3Zy99wp1V(BAbtUpYe`1dlcP z_B&)+nvU<5#M~&wFm)Y?IFB?&MzBn98}_F(#mB_0!+2&erE8G`>>_&2?yP-6JV0g! zwDSJ=VlPp$F!iz{uQ8GA2T&RfdoVt*V!Y*Qj=3cKN}5)kz9Bk?^A`{PJX&h!9ho#} z*7eRCN!NlB+o~{{oWfm^w*T~^cFhMcBKDhxFkn?K&nUWGvQ#gLb{DaotW#rH+wHPwpCHV?LcV=Z}a!?`rc&{xN^eL-jbIBU+k#=!ab)U6+C@H}2a z@l&+!i)_8L3B121tQd#1IRa4U91BO;FY%X)b&{Sl8Bhtcekv7YLoweP#yGVSTQsleR#DR@(lQE!0a)!v+@#e8?vXdet8#H>oL*9WXNExQH=Y_SVY?MoLo2f>m84|DZanx zwlBOVXhr^njU>nBxk7P+Q7FFzyEUo*(-5Vs2=XwYpS1}O0^SUM{ybYgq&UpNUsI}j z7TY2+et{f=m9$S-4;xy)>C{X9HD?|Y&IZmMP%GB3v6gVODzL1F;?{-nXOFo9?jwjk1Wv_at{vA=|b<@@{tyy~$CD{WutpK$+5Bc}8J=W~|J!Qk4pv z&CvYZlYqxxW{hCj;^Af#+{;$-yJ@CQQ>M%CgCn{tFl<~At_K%}ll=450depEAQqZr zF|(UIiWffl!nmEY6G>V Date: Fri, 31 Jan 2025 16:43:45 +0530 Subject: [PATCH 11/39] Fix the configuration issue --- examples/kafka-hub/helm/hub/templates/secret.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kafka-hub/helm/hub/templates/secret.yaml b/examples/kafka-hub/helm/hub/templates/secret.yaml index 49f172f1..00f1d909 100644 --- a/examples/kafka-hub/helm/hub/templates/secret.yaml +++ b/examples/kafka-hub/helm/hub/templates/secret.yaml @@ -21,6 +21,6 @@ metadata: name: {{ .name }} type: Opaque data: - {{ .subPath }}: {{ .Files.Get .filePath | b64enc }} + {{ .subPath }}: {{ $.Files.Get .filePath | b64enc }} --- {{- end }} From c65e664a4e54395fd7033922f18537df0d9a8ccc Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 31 Jan 2025 16:44:41 +0530 Subject: [PATCH 12/39] Restructure the code --- examples/kafka-hub/helm/hub/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kafka-hub/helm/hub/templates/deployment.yaml b/examples/kafka-hub/helm/hub/templates/deployment.yaml index 1555e252..f8de2089 100644 --- a/examples/kafka-hub/helm/hub/templates/deployment.yaml +++ b/examples/kafka-hub/helm/hub/templates/deployment.yaml @@ -45,7 +45,7 @@ spec: mountPath: {{ .mountPath }} subPath: {{ .subPath }} readOnly: true - {{- end }} + {{- end }} volumes: - name: "config-config-map-volume" From 981997a9eb32143d5502f844fe58ee006c4c7a00 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 31 Jan 2025 16:47:25 +0530 Subject: [PATCH 13/39] Add support to mount secrets to the consolidator pod --- .../kafka-client/kafka-client.keystore.jks | Bin 0 -> 2756 bytes .../kafka-client/kafka-client.trustStore.jks | Bin 0 -> 3776 bytes .../consolidator/templates/deployment.yaml | 12 ++++++++ .../helm/consolidator/templates/secret.yaml | 26 ++++++++++++++++++ .../kafka-hub/helm/consolidator/values.yaml | 9 ++++++ 5 files changed, 47 insertions(+) create mode 100644 examples/kafka-hub/helm/consolidator/_resources/kafka-client/kafka-client.keystore.jks create mode 100644 examples/kafka-hub/helm/consolidator/_resources/kafka-client/kafka-client.trustStore.jks create mode 100644 examples/kafka-hub/helm/consolidator/templates/secret.yaml diff --git a/examples/kafka-hub/helm/consolidator/_resources/kafka-client/kafka-client.keystore.jks b/examples/kafka-hub/helm/consolidator/_resources/kafka-client/kafka-client.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..6c02a44d5caec36d14c7b682619f159e8832ac0e GIT binary patch literal 2756 zcma)8X*d*&7M>Zi8bT;b7&RzO8cR{OEZLRpTgbleC1jfnvg<1u`Y^JLU6wM&$PzUQ zgX~`vWyu)pAX3*|p1$Y)y7$L9=Q;0ke!kDa;@I0kKsGFn-5ZLKPS8u(;RM2fg*bLQ zFpk~o5H`c&AT9qzK`OyGNZBF$?63-;$p5%F*nyxz9OTL&sDgF>?SOD%ZLmszuN>I3 zU_R&3Dwa{Q^%7`gVBqn!4~P#@2_O(iiW>w(VL71ie_sS4AONfg6ycPh2Xtct1Es;- z`S-f)OA#4=^m8mi#gS zv#dH3VCAidns-tG0_)uv2$=0It8p8Yk~t&S-~skj;VY1sx3dd3w5-%j_t`&|-=bWE z(~H~3mr@HX?Zq`z@!`lC(Kd`cQDHtlahA|&1Eyz4l?VkjBR=pvaGpul`g2z749|9w z--7u0=OHHI^Q4(wf$-K)<8iNS^D{!w6WS|LcoU319c_{(*b8%w&N&*p8n&tM*r}w& zt?MDE3p>$x5W++t3$!v&*>t_IKA&=QsX(L{=+f+?@|30YfRa@B|#c7?)m6A}m9>84sR4x)0-9OurUREP~ zic?v-?1;V*L3MKH99sNbL>eON8hqJe7jZpij7IDJV)BCceJ(lH+z)g7{FdO-g4Kx1 z8aFY%gm~Zgny8Btg3KAL>7{_V&Jq@eVpy@wtl?JtCqRfU*USdJ)~)=y>8%vzTg+Hh zkotOFZ$hO?2D#eD!IN@J^YTumq4{15KyqxQYoZGMyEzG*8UQRd?a zSAK(aE5o>}qHKlZS39|6bbCjZ#0yC8db{~upVCD87@wXUnw~ym$Ru@1dzeC}Q|XiW zwU%53RtSKk-9f|5BcVxwIx4OWTmpl8NvUSvY+^3yp|`E>n16$>O${lms`_YpjC%=7 zTmr7c?43aZi%^Yy&G6jmZuR0eWKQ-Bh3Q+4+SNt-|9%r6mBbPT>y{Nx?(1f0ZG zjKsCCy*K9qIeQM7re)ejZ#0E{_1qXc$q)>uuLd}tIMVU0x?}B>dK-1R61gG>kb+)Vmj@JFR>PHroS#6D~SNpsc=in)nt=5jM+I2@R} z1_PWfzI{S|Il7!z&3G}~wq*U8<-#(q%Y|jaDKv{)IWgk%qPiX)u^yF#x=T;S$=Sx@z<|J>S34#Cw3fJ|LTui;z?tOMPl}_8x3A2DvbD;C$nDK_8anC)va$|7?!9^naBf-4V{NmyCJ6AO z952q#%#V@Y^qXclgeM(G-QCEPsju6}-?MdwpZ7|Mo^9_pzjblp14ik?QzW781ZSkc z8{dg7#eJUi&t>Y|5zQK7YkNaSV8jQKB9$+L-Zc7MIX6(46TZYKzMVrB>JGK{;Y+l3 zHlLW#c6?=S7FZ(TvS0D~%ccEz4 z1~1@yz9aa0k(gv%Xvnb?RBFaSqZE^x0n6ewx9#|L5_d&7@*r?S1)?p_waV@TuU|WM zX~6U~>w}R+rIE<)5?`wFJ=7SzBIGDRTWe9n+tH^cRxBqXE4U!5G{_P4#eDk$>1cR4K3@MeQxs6=EM+!U zZOgQ$M@P?7oKAzLBqtg?p6@rFh1cR5JqV(?eX&Dp7p4k$`+&MJ4`t_0G1@s&R&sHz zs!R)i{*NQeLw{INJ$&(0T8u_6w77sqR_=YH zMSEZEaH&FVdGxO$K$#8j6`ljVFWHLHGXrMnnv~>mj5y+S#w9FqCU?$=O3;&vNL#N6 z)Z!~^SMX{^w&{Ig-#^fE9toUuHV^fO?rgHj6r`49HiFvRmX+KpOYLr)HVR&x;QAWeH% zC~@}TORLp zVxt-fg_Hb(4%Q?;!9DW+QHm1e&(Q|2n?)V+J6z~IFcA7c*6{AEa`5G(t=de=iHqAK ztn@j{2pMk4*|K9JrPew-`2pUJ^R0TS)(c##@kna}Ji4v_TDnsooIN)EaYct9DSO(e zFmnTIjFrW5{{HMhKnMVg3RKjPYx2L)M^^oAaq-1Nw7`?JqAX8C<7s|~J4VfZ8O(QY a&R1B-us@1i>qs(aq3zh8|7-4_5%dpIH|{wA literal 0 HcmV?d00001 diff --git a/examples/kafka-hub/helm/consolidator/_resources/kafka-client/kafka-client.trustStore.jks b/examples/kafka-hub/helm/consolidator/_resources/kafka-client/kafka-client.trustStore.jks new file mode 100644 index 0000000000000000000000000000000000000000..9c71e77fa120529ab5dbe2aba7ba831d76a03fbd GIT binary patch literal 3776 zcma)9Ra6uTv>k>4W{4pNq+1$>hD&!NFm!i^3?U#ALrUreK|n%~8d?+_LPDfN=>Z8T zk?xM?i*@gMU+;bFbJp7Doc+1iK5!JY4TysaM?qcjA-wUL@#iEs1UN+~s2Kh^kxX z@?|!D__XP2_IrJXA;@k*odFQYMFzxSffM6{|636V!2`e<@ger{nm8`FARJy0*`x?s zC%0^SV^cH3Org1W0fYj5!x1#R1(u90ejMvH()vUV66H|#U>)0YUUXXNux={cYbK~# z#b)n*24jgTtDKHI*KkdKzN^%j+m=148I^hx<>N9u=cf}NU&pk&y|V(Tg$_5-_O+!k z{P<24dIsx99BEtQZWMB-rt zWl%pOY1a=fjwM2I&&uzHD*U6A&d0>P_eO*5lPvT6RVkMnvIL}Wp6I* z)b*FL#i`swzc0iNp(jOFNtj=l1`x_0651vXH?URBRrahH{GmKaG6$J;?}wp7mqm^LU)#HBv2FUStk z%LGhVyu-hqazfxeyXeTyHDIDkzsUG_Sz^fIGB{z|TXcI`-i>MUGoCZ#kh$Je+`4bI z&=8#Ra#cDQe$W*6sN7bP$Uw$aCy~C4V-?XCyeV`>QPPnj8k_~9>UYT5{##MD*pI5p zi$qF)r4V|T-`vxXbM`RlQbw0!rE#)6M)t|rC-fH*0cp|IE(JgPXPF~6c2(m(SRMTy z8D9y)A^s>v>T;$vkpKgxN|!g%_g*J~TiXOf6tT(2E%F14lp;R(2DR6Uzm^X(q(_UD zW^1L%6(B#5Ig>cl#am)%@0PP~NNqx7f9nTB9wHY@86}=p-y$Y?Vu|WGOgjIwX0P{DAxPrDpvkLZi~f@G9!p2rA2G!x9r(M)l3wr zP`koxL(h)_+0iXFKlNfY>=K!cC+pmd-Q&Ge*cO+JoP~;+SZ2UPav+Pp4acKyM_s7i z^VS7)EE~XlVeIL4^YXz!y&YZha4MWz%8lJR+-L{lYrW|^4EXkU4BmeleUyQ*6K zvnOl;cMs>p2&yK<=fTHd7;vu9veBlPf)qh`TWVqRmdr>O@%3DgYzKocbRKJlBr(f5 zx*PIl)TVdcY^_ozzJZT3v?Qt^3P7|y!Dd}#V9JextKb)X9G%Im2#o3UmHnAjo0HM9 zw=`>r8NHh7JSdF8)-mt<;-1+Was6SZ@VuKl`0QIbNd8b^Fsj7%v(mDC!ZSPO^{+uD zeJd73*K|LpSQHov>q+H|EBEh?47>?tf3LT%Zx>@va3k7jH4@Fp`>A=g}PRf^5-8CA_BaA70r*8`*V!K~BN! zbkk#gmv2vVrs=x1h49`5$K=Eu|2&58&05Oj-Nu!!cQn9u)PF6GL}T5LlX`)x8QRIh z=5Wz}>=HsH{E&$Q;0161xc*la05}1>0UiJsfG;2rAn*rBKoB4R5C~`e_Xs9T1)?za z@Nr=gmV}GJC8eY#MWx|zI0_v7&k_NC5ekg_L-_)60Dqd>zZ0DQ%f7LkPN7nTiBsq@ zNhQHhs9$0lq0Ilsz871QMO@5&E|<$6RX5q3pGTnx4k{}!A6x-e%1HwcL~5r`@vf-X zhqx+dtgr}&#XXp6iF{ToN1^iwkl4#LpJx*3Uzj^oj{v1>r9uf>?0amN4<+=IMGqu% z=H#QLZ|#fjSE3IFkUX-K>H5;kpLIk(HH~I#Hkm~CkW>sT5cu*2XVvmWe`~ycx=x~u zoPva9rm*CfM(xC#+YguMoPHRIHN7R*rW{b{!y=Y<8hMZVoCIxEZOjS_Y>@E+Swu}7(O~qL)FrNiq}fhd z#7=QB#*jXZaW(RXpgPGH8kOr)ekQicNNzx2ul3*LaU}vb)c}~Z;Dx?h#?3=DW0nf` z)D{MxgcsVbT%taCd@)DTfgK91G-0ZB-z12Km>(sl>sYLRU7j{%pG|PwkHaT)mC(zm zdQO4zkh`faSV}!fA+~IHsm@T@X?G00g);#fHfv)Aa;_t<1<=^|RX}svomD+?$@gCmUj{E#l=9)}9q6wU_)yWnW zzJzpn)e|Jl555&47G8~&7~8F`cH_*Ab>bcJK3hJJUs&9<+oW^uJUJ~B5L;mt9$5futdF*_Jega7%^VB3S~&k!q`M^N zr8dJp+krV^xy|iqzNZqi-uYwui>LVK`M}W_K~CM6ILh8z-?9V& zf}OzTg|?^&R#H3IE$elbn_}q~Qy{7(wBEk6X?;UI!R_Xa(@;E)ewbsTfk+U8S%`;7 zc1a2ER|lbepr2QlP?vkLv`NL4t@2pp~QRTl4gsv{|W&<%2CBN>A(b4 z(=L4v+zt9PmXs~x;}Fdn>Zz!0*1PC*CoRcDs!TVNy;!*YHUN!(VUeUp86Ei{Rr3R1i)YPxZaDIAm+M;Usc&P zR{@!*iu}u0FkkG>l=It{WX~h*d{ID{?+lyZ`)aR|OfSw>IIM%71Vs}ymV6d()4_i%KQBE;bJdY|Sj z&4Mu*CktJC2hx+&>6+>NuMs0xcf*f(Dw)du#;K8iN*RBZQ4rOTKEWkuq;;P;K%QT& zCNif3UOqHMoDts!)+k5w^O=NC^Aj+JEfEP0^*sfgFjj<&$oE(ss*@x0f>2!rVJsg$ew{z%KF7TK#D3Zy99wp1V(BAbtUpYe`1dlcP z_B&)+nvU<5#M~&wFm)Y?IFB?&MzBn98}_F(#mB_0!+2&erE8G`>>_&2?yP-6JV0g! zwDSJ=VlPp$F!iz{uQ8GA2T&RfdoVt*V!Y*Qj=3cKN}5)kz9Bk?^A`{PJX&h!9ho#} z*7eRCN!NlB+o~{{oWfm^w*T~^cFhMcBKDhxFkn?K&nUWGvQ#gLb{DaotW#rH+wHPwpCHV?LcV=Z}a!?`rc&{xN^eL-jbIBU+k#=!ab)U6+C@H}2a z@l&+!i)_8L3B121tQd#1IRa4U91BO;FY%X)b&{Sl8Bhtcekv7YLoweP#yGVSTQsleR#DR@(lQE!0a)!v+@#e8?vXdet8#H>oL*9WXNExQH=Y_SVY?MoLo2f>m84|DZanx zwlBOVXhr^njU>nBxk7P+Q7FFzyEUo*(-5Vs2=XwYpS1}O0^SUM{ybYgq&UpNUsI}j z7TY2+et{f=m9$S-4;xy)>C{X9HD?|Y&IZmMP%GB3v6gVODzL1F;?{-nXOFo9?jwjk1Wv_at{vA=|b<@@{tyy~$CD{WutpK$+5Bc}8J=W~|J!Qk4pv z&CvYZlYqxxW{hCj;^Af#+{;$-yJ@CQQ>M%CgCn{tFl<~At_K%}ll=450depEAQqZr zF|(UIiWffl!nmEY6G>V Date: Fri, 31 Jan 2025 16:59:58 +0530 Subject: [PATCH 14/39] Migrate hub deployment to a statefulset --- .../hub/templates/{deployment.yaml => statefulset.yaml} | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) rename examples/kafka-hub/helm/hub/templates/{deployment.yaml => statefulset.yaml} (87%) diff --git a/examples/kafka-hub/helm/hub/templates/deployment.yaml b/examples/kafka-hub/helm/hub/templates/statefulset.yaml similarity index 87% rename from examples/kafka-hub/helm/hub/templates/deployment.yaml rename to examples/kafka-hub/helm/hub/templates/statefulset.yaml index f8de2089..c9693299 100644 --- a/examples/kafka-hub/helm/hub/templates/deployment.yaml +++ b/examples/kafka-hub/helm/hub/templates/statefulset.yaml @@ -15,7 +15,7 @@ # under the License. apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ .Release.Name }}-deployment labels: @@ -36,6 +36,13 @@ spec: imagePullPolicy: {{ .Values.deployment.image.pullPolicy }} ports: - containerPort: {{ .Values.deployment.config.port }} + env: + - name: "SERVER_ID" + valueFrom: + fieldRef: + fieldPath: {{ .Values.deployment.config.server_id }} + - name: "BALLERINA_MAX_POOL_SIZE" + value: "100" volumeMounts: - name: "config-config-map-volume" mountPath: "/home/ballerina" From f56b38e2855908dcf608f798c43de8000de0d2fb Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 31 Jan 2025 17:05:53 +0530 Subject: [PATCH 15/39] Update consolidator URL --- examples/kafka-hub/helm/hub/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kafka-hub/helm/hub/values.yaml b/examples/kafka-hub/helm/hub/values.yaml index 15a89e44..73b35da8 100644 --- a/examples/kafka-hub/helm/hub/values.yaml +++ b/examples/kafka-hub/helm/hub/values.yaml @@ -40,7 +40,7 @@ deployment: config: port: 9000 server_id: "hub-1" - state_snapshot_endpoint: "http://consolidator:10001" + state_snapshot_endpoint: "http://ballerina-consolidator-deployment:10001" retryable_status_codes: [500, 502, 503] logLevel: "ERROR" ssl_keystoreName: "hub.keystore.jks" From 06af2b4f3b425edb5b81aed19c6a2f3885e2c1ef Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Mon, 3 Feb 2025 09:44:57 +0530 Subject: [PATCH 16/39] Fix secret name configuration --- examples/kafka-hub/helm/consolidator/values.yaml | 6 +++--- examples/kafka-hub/helm/hub/values.yaml | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/kafka-hub/helm/consolidator/values.yaml b/examples/kafka-hub/helm/consolidator/values.yaml index 661def69..d54c9924 100644 --- a/examples/kafka-hub/helm/consolidator/values.yaml +++ b/examples/kafka-hub/helm/consolidator/values.yaml @@ -21,12 +21,12 @@ deployment: pullPolicy: IfNotPresent tag: 7.0.0 secrets: - - name: broker_truststore + - name: broker-truststore mountPath: "/home/ballerina/resources/brokercerts/client-truststore.jks" subPath: "client-truststore.jks" filePath: "./_resources/kafka-client/kafka-client.trustStore.jks" - - name: broker_keystore - mountPath: "/home/ballerina/resources/brokercerts/client-truststore.jks" + - name: broker-keystore + mountPath: "/home/ballerina/resources/brokercerts/client-keystore.jks" subPath: "client-keystore.jks" filePath: "./_resources/kafka-client/kafka-client.keystore.jks" config: diff --git a/examples/kafka-hub/helm/hub/values.yaml b/examples/kafka-hub/helm/hub/values.yaml index 73b35da8..4d7407d1 100644 --- a/examples/kafka-hub/helm/hub/values.yaml +++ b/examples/kafka-hub/helm/hub/values.yaml @@ -21,19 +21,19 @@ deployment: pullPolicy: IfNotPresent tag: 12.0.0 secrets: - - name: hub_keystore + - name: hub-keystore mountPath: "/home/ballerina/resources/hub.keystore.jks" subPath: "hub.keystore.jks" filePath: "./_resources/hub-1.keystore.jks" - - name: idp_truststore + - name: idp-truststore mountPath: "/home/ballerina/resources/client-truststore.jks" subPath: "client-truststore.jks" filePath: "./_resources/idp/client-truststore.jks" - - name: broker_truststore + - name: broker-truststore mountPath: "/home/ballerina/resources/brokercerts/client-truststore.jks" subPath: "client-truststore.jks" filePath: "./_resources/kafka-client/kafka-client.trustStore.jks" - - name: broker_keystore + - name: broker-keystore mountPath: "/home/ballerina/resources/brokercerts/client-truststore.jks" subPath: "client-keystore.jks" filePath: "./_resources/kafka-client/kafka-client.keystore.jks" From 262a9ac2f840859a45648ab35343d7ebd0252d98 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Mon, 3 Feb 2025 10:08:49 +0530 Subject: [PATCH 17/39] Refactor chart configurations --- examples/kafka-hub/helm/consolidator/Chart.yaml | 2 +- examples/kafka-hub/helm/consolidator/values.yaml | 4 ++-- examples/kafka-hub/helm/hub/values.yaml | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/kafka-hub/helm/consolidator/Chart.yaml b/examples/kafka-hub/helm/consolidator/Chart.yaml index c393918c..1f3fc69c 100644 --- a/examples/kafka-hub/helm/consolidator/Chart.yaml +++ b/examples/kafka-hub/helm/consolidator/Chart.yaml @@ -18,4 +18,4 @@ apiVersion: v2 appVersion: 7.0.0 description: A Helm chart for the deployment of WSO2 Websubhub Consolidator Deployment name: consolidator -version: 0.1.24 +version: 0.1.0 diff --git a/examples/kafka-hub/helm/consolidator/values.yaml b/examples/kafka-hub/helm/consolidator/values.yaml index d54c9924..179d6135 100644 --- a/examples/kafka-hub/helm/consolidator/values.yaml +++ b/examples/kafka-hub/helm/consolidator/values.yaml @@ -21,11 +21,11 @@ deployment: pullPolicy: IfNotPresent tag: 7.0.0 secrets: - - name: broker-truststore + - name: ballerina-consolidator-broker-truststore mountPath: "/home/ballerina/resources/brokercerts/client-truststore.jks" subPath: "client-truststore.jks" filePath: "./_resources/kafka-client/kafka-client.trustStore.jks" - - name: broker-keystore + - name: ballerina-consolidator-broker-keystore mountPath: "/home/ballerina/resources/brokercerts/client-keystore.jks" subPath: "client-keystore.jks" filePath: "./_resources/kafka-client/kafka-client.keystore.jks" diff --git a/examples/kafka-hub/helm/hub/values.yaml b/examples/kafka-hub/helm/hub/values.yaml index 4d7407d1..91aeaed1 100644 --- a/examples/kafka-hub/helm/hub/values.yaml +++ b/examples/kafka-hub/helm/hub/values.yaml @@ -21,19 +21,19 @@ deployment: pullPolicy: IfNotPresent tag: 12.0.0 secrets: - - name: hub-keystore + - name: ballerina-websubhub-hub-keystore mountPath: "/home/ballerina/resources/hub.keystore.jks" subPath: "hub.keystore.jks" filePath: "./_resources/hub-1.keystore.jks" - - name: idp-truststore + - name: ballerina-websubhub-idp-truststore mountPath: "/home/ballerina/resources/client-truststore.jks" subPath: "client-truststore.jks" filePath: "./_resources/idp/client-truststore.jks" - - name: broker-truststore + - name: ballerina-websubhub-broker-truststore mountPath: "/home/ballerina/resources/brokercerts/client-truststore.jks" subPath: "client-truststore.jks" filePath: "./_resources/kafka-client/kafka-client.trustStore.jks" - - name: broker-keystore + - name: ballerina-websubhub-broker-keystore mountPath: "/home/ballerina/resources/brokercerts/client-truststore.jks" subPath: "client-keystore.jks" filePath: "./_resources/kafka-client/kafka-client.keystore.jks" From 6e53225a893224ac33c48366a5a8222a13140bdd Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Mon, 3 Feb 2025 10:23:13 +0530 Subject: [PATCH 18/39] Fix secret values not getting populated properly --- examples/kafka-hub/helm/consolidator/values.yaml | 4 ++-- examples/kafka-hub/helm/hub/values.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/kafka-hub/helm/consolidator/values.yaml b/examples/kafka-hub/helm/consolidator/values.yaml index 179d6135..e3945ec2 100644 --- a/examples/kafka-hub/helm/consolidator/values.yaml +++ b/examples/kafka-hub/helm/consolidator/values.yaml @@ -24,11 +24,11 @@ deployment: - name: ballerina-consolidator-broker-truststore mountPath: "/home/ballerina/resources/brokercerts/client-truststore.jks" subPath: "client-truststore.jks" - filePath: "./_resources/kafka-client/kafka-client.trustStore.jks" + filePath: "_resources/kafka-client/kafka-client.trustStore.jks" - name: ballerina-consolidator-broker-keystore mountPath: "/home/ballerina/resources/brokercerts/client-keystore.jks" subPath: "client-keystore.jks" - filePath: "./_resources/kafka-client/kafka-client.keystore.jks" + filePath: "_resources/kafka-client/kafka-client.keystore.jks" config: port: 10001 kafka: diff --git a/examples/kafka-hub/helm/hub/values.yaml b/examples/kafka-hub/helm/hub/values.yaml index 91aeaed1..a42c6ea4 100644 --- a/examples/kafka-hub/helm/hub/values.yaml +++ b/examples/kafka-hub/helm/hub/values.yaml @@ -24,19 +24,19 @@ deployment: - name: ballerina-websubhub-hub-keystore mountPath: "/home/ballerina/resources/hub.keystore.jks" subPath: "hub.keystore.jks" - filePath: "./_resources/hub-1.keystore.jks" + filePath: "_resources/hub-1.keystore.jks" - name: ballerina-websubhub-idp-truststore mountPath: "/home/ballerina/resources/client-truststore.jks" subPath: "client-truststore.jks" - filePath: "./_resources/idp/client-truststore.jks" + filePath: "_resources/idp/client-truststore.jks" - name: ballerina-websubhub-broker-truststore mountPath: "/home/ballerina/resources/brokercerts/client-truststore.jks" subPath: "client-truststore.jks" - filePath: "./_resources/kafka-client/kafka-client.trustStore.jks" + filePath: "_resources/kafka-client/kafka-client.trustStore.jks" - name: ballerina-websubhub-broker-keystore mountPath: "/home/ballerina/resources/brokercerts/client-truststore.jks" subPath: "client-keystore.jks" - filePath: "./_resources/kafka-client/kafka-client.keystore.jks" + filePath: "_resources/kafka-client/kafka-client.keystore.jks" config: port: 9000 server_id: "hub-1" From 3bd177ddebc50993d59e104ba4d48db383e69c3d Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Thu, 6 Feb 2025 23:14:36 +0530 Subject: [PATCH 19/39] Migrate the consolidator code-base to SL U10 --- examples/kafka-hub/consolidator/Ballerina.toml | 1 + examples/kafka-hub/consolidator/consolidator_service.bal | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/kafka-hub/consolidator/Ballerina.toml b/examples/kafka-hub/consolidator/Ballerina.toml index b6c9f99e..f012fc03 100644 --- a/examples/kafka-hub/consolidator/Ballerina.toml +++ b/examples/kafka-hub/consolidator/Ballerina.toml @@ -2,6 +2,7 @@ org = "example" name = "consolidatorService" version = "0.1.0" +distribution = "2201.10.3" [build-options] observabilityIncluded = true diff --git a/examples/kafka-hub/consolidator/consolidator_service.bal b/examples/kafka-hub/consolidator/consolidator_service.bal index db377f0f..ebeb76a5 100644 --- a/examples/kafka-hub/consolidator/consolidator_service.bal +++ b/examples/kafka-hub/consolidator/consolidator_service.bal @@ -37,8 +37,8 @@ http:Service consolidatorService = service object { isolated function consolidateSystemState() returns error? { do { while true { - kafka:ConsumerRecord[] records = check conn:websubEventConsumer->poll(config:POLLING_INTERVAL); - foreach kafka:ConsumerRecord currentRecord in records { + kafka:BytesConsumerRecord[] records = check conn:websubEventConsumer->poll(config:POLLING_INTERVAL); + foreach kafka:BytesConsumerRecord currentRecord in records { string lastPersistedData = check string:fromBytes(currentRecord.value); error? result = processPersistedData(lastPersistedData); if result is error { From 42bcb7a3f052bddff62a7a05b2add6ce4652fa34 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Thu, 6 Feb 2025 23:15:28 +0530 Subject: [PATCH 20/39] Remove unwanted files --- examples/kafka-hub/consolidator/docker/alpine/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kafka-hub/consolidator/docker/alpine/Dockerfile b/examples/kafka-hub/consolidator/docker/alpine/Dockerfile index 3fca47e6..650ca32b 100644 --- a/examples/kafka-hub/consolidator/docker/alpine/Dockerfile +++ b/examples/kafka-hub/consolidator/docker/alpine/Dockerfile @@ -58,4 +58,4 @@ WORKDIR ${USER_HOME} ENV JAVA_OPTS="-Djava.util.prefs.systemRoot=${USER_HOME}/.java -Djava.util.prefs.userRoot=${USER_HOME}/.java/.userPrefs" \ WORKING_DIRECTORY=${USER_HOME} -CMD java -jar ${JAR_FILE_PATH} +CMD java -jar "${JAR_FILE_PATH}" From 87886730cb27941f34cd105e46c0e4a7f7cbabb1 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Thu, 6 Feb 2025 23:15:56 +0530 Subject: [PATCH 21/39] Remove unwanted files --- .../consolidator/docker/alpine/Dockerfile | 61 ------------------- 1 file changed, 61 deletions(-) delete mode 100644 examples/kafka-hub/consolidator/docker/alpine/Dockerfile diff --git a/examples/kafka-hub/consolidator/docker/alpine/Dockerfile b/examples/kafka-hub/consolidator/docker/alpine/Dockerfile deleted file mode 100644 index 650ca32b..00000000 --- a/examples/kafka-hub/consolidator/docker/alpine/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -# ------------------------------------------------------------------------ -# -# Copyright (c) 2022, WSO2 LLC. (http://www.wso2.com). All Rights Reserved. -# -# This software is the property of WSO2 LLC. and its suppliers, if any. -# Dissemination of any information or reproduction of any material contained -# herein in any form is strictly forbidden, unless permitted by WSO2 -# expressly. You may not alter or remove any copyright or other notice from -# copies of this content. -# -# ------------------------------------------------------------------------ - -ARG BASE_IMAGE_DIGEST -FROM adoptopenjdk/openjdk11@${BASE_IMAGE_DIGEST} -LABEL maintainer="WSO2 Docker Maintainers " - -# set Docker image build arguments -# build arguments for user/group configurations -ARG USER=websubhub -ARG USER_ID=10001 -ARG USER_GROUP=wso2 -ARG USER_GROUP_ID=10001 -ARG USER_HOME=/home/${USER} -# build arguments for WSO2 product installation -ARG JAR_FILE_PATH=consolidatorService.jar - - -# build argument for MOTD -ARG MOTD="\n\ -Welcome to WSO2 Docker resources.\n\ ------------------------------------- \n\ -This Docker container comprises of a WSO2 product, running with its latest GA release \n\ -which is under the Apache License, Version 2.0. \n\ -Read more about Apache License, Version 2.0 here @ http://www.apache.org/licenses/LICENSE-2.0.\n" - -# create the non-root user and group and set MOTD login message -RUN \ - addgroup -g ${USER_GROUP_ID} -S ${USER_GROUP} && adduser -u ${USER_ID} -S ${USER} \ - -G ${USER_GROUP} -h ${USER_HOME} \ - && echo '[ ! -z "${TERM}" -a -r /etc/motd ] && cat /etc/motd' >> /etc/bash.bashrc; echo "${MOTD}" > /etc/motd - -# create Java prefs dir -# this is to avoid warning logs printed by FileSystemPreferences class -RUN \ - mkdir -p ${USER_HOME}/.java/.systemPrefs \ - && mkdir -p ${USER_HOME}/.java/.userPrefs \ - && chmod -R 755 ${USER_HOME}/.java \ - && chown -R ${USER}:${USER_GROUP} ${USER_HOME}/.java - -# copy init script to user home -COPY --chown=${USER}:${USER_GROUP} ${JAR_FILE_PATH} ${USER_HOME}/ - -# set the user and work directory -USER ${USER_ID} -WORKDIR ${USER_HOME} - -# set environment variables -ENV JAVA_OPTS="-Djava.util.prefs.systemRoot=${USER_HOME}/.java -Djava.util.prefs.userRoot=${USER_HOME}/.java/.userPrefs" \ - WORKING_DIRECTORY=${USER_HOME} - -CMD java -jar "${JAR_FILE_PATH}" From b56ba94fcb16b072d883b953ee744dc32c60ad57 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Thu, 6 Feb 2025 23:16:12 +0530 Subject: [PATCH 22/39] Add a dockerfile for the consolidator --- examples/kafka-hub/consolidator/Dockerfile | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 examples/kafka-hub/consolidator/Dockerfile diff --git a/examples/kafka-hub/consolidator/Dockerfile b/examples/kafka-hub/consolidator/Dockerfile new file mode 100644 index 00000000..02729027 --- /dev/null +++ b/examples/kafka-hub/consolidator/Dockerfile @@ -0,0 +1,42 @@ +ARG BASE_IMAGE_DIGEST=sha256:6bb29faf5333dcc2034cad9d29abd0832846c8d961143c70a4c0e504fdfc9543 +FROM eclipse-temurin:17.0.14_7-jdk-ubi9-minimal@${BASE_IMAGE_DIGEST} +LABEL maintainer="WSO2 Docker Maintainers " + +# set Docker image build arguments +# build arguments for user/group configurations +ARG USER=consolidator +ARG USER_ID=10001 +ARG USER_GROUP=wso2 +ARG USER_GROUP_ID=10001 +ARG USER_HOME=/home/${USER} + +# build arguments for WSO2 product installation +ARG JAR_FILE_PATH=target/bin/consolidatorService.jar + +# build argument for MOTD +ARG MOTD="\n\ +Welcome to WSO2 Docker resources.\n\ +------------------------------------ \n\ +This Docker container comprises of a WSO2 product, running with its latest GA release \n\ +which is under the Apache License, Version 2.0. \n\ +Read more about Apache License, Version 2.0 here @ http://www.apache.org/licenses/LICENSE-2.0.\n" + +# create the non-root user and group and set MOTD login message +RUN \ + addgroup -g ${USER_GROUP_ID} -S ${USER_GROUP} && adduser -u ${USER_ID} -S ${USER} \ + -G ${USER_GROUP} -h ${USER_HOME} \ + && echo '[ ! -z "${TERM}" -a -r /etc/motd ] && cat /etc/motd' >> /etc/bash.bashrc; echo "${MOTD}" > /etc/motd + +# copy init script to user home +COPY --chown=${USER}:${USER_GROUP} ${JAR_FILE_PATH} ${USER_HOME}/ + +# set the user and work directory +USER ${USER_ID} +WORKDIR ${USER_HOME} + +# set environment variables +ENV JAVA_OPTS="-Djava.util.prefs.systemRoot=${USER_HOME}/.java -Djava.util.prefs.userRoot=${USER_HOME}/.java/.userPrefs" \ + WORKING_DIRECTORY=${USER_HOME} + +# ENTRYPOINT [ "java" , "-jar", "consolidatorService.jar" ] +CMD java -jar consolidatorService.jar From 589661f921ee6e28de765defb9f1085db2387649 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Thu, 6 Feb 2025 23:17:21 +0530 Subject: [PATCH 23/39] Fix helm configurations for consolidator --- .../helm/consolidator/confs/Config.toml | 6 +++--- .../helm/consolidator/templates/conf.yaml | 2 +- .../helm/consolidator/templates/deployment.yaml | 10 ++++++++-- .../helm/consolidator/templates/secret.yaml | 4 +++- .../kafka-hub/helm/consolidator/values.yaml | 17 ++++++++--------- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/examples/kafka-hub/helm/consolidator/confs/Config.toml b/examples/kafka-hub/helm/consolidator/confs/Config.toml index 9d3e53ab..cfafecb1 100644 --- a/examples/kafka-hub/helm/consolidator/confs/Config.toml +++ b/examples/kafka-hub/helm/consolidator/confs/Config.toml @@ -34,8 +34,8 @@ GRACEFUL_CLOSE_PERIOD = 5.0 CONSOLIDATOR_HTTP_ENDPOINT_PORT = {{ .Values.deployment.config.port }} # The MTLS configurations related to Kafka connection -[kafkaHub.config.KAFKA_MTLS_CONFIG] -cert.path = "/home/ballerina/resources/brokercerts/{{ .Values.deployment.config.kafka.truststore_name }}" +[consolidatorService.config.KAFKA_MTLS_CONFIG] +cert.path = "/home/consolidator/resources/brokercerts/{{ .Values.deployment.config.kafka.truststore_name }}" cert.password = {{ .Values.deployment.config.kafka.truststore_password | quote }} -key.keyStore.path = "/home/ballerina/resources/brokercerts/{{ .Values.deployment.config.kafka.keystore_name }}" +key.keyStore.path = "/home/consolidator/resources/brokercerts/{{ .Values.deployment.config.kafka.keystore_name }}" key.keyStore.password = {{ .Values.deployment.config.kafka.keystore_password | quote }} diff --git a/examples/kafka-hub/helm/consolidator/templates/conf.yaml b/examples/kafka-hub/helm/consolidator/templates/conf.yaml index df7f8fc9..e6de4afc 100644 --- a/examples/kafka-hub/helm/consolidator/templates/conf.yaml +++ b/examples/kafka-hub/helm/consolidator/templates/conf.yaml @@ -20,4 +20,4 @@ metadata: name: consolidator-svc-cm namespace : {{ .Release.Namespace }} data: - config.toml: {{ tpl (.Files.Get "confs/Config.toml") . | quote }} + Config.toml: {{ tpl (.Files.Get "confs/Config.toml") . | quote }} diff --git a/examples/kafka-hub/helm/consolidator/templates/deployment.yaml b/examples/kafka-hub/helm/consolidator/templates/deployment.yaml index 29af0e33..cf74ae56 100644 --- a/examples/kafka-hub/helm/consolidator/templates/deployment.yaml +++ b/examples/kafka-hub/helm/consolidator/templates/deployment.yaml @@ -36,14 +36,17 @@ spec: imagePullPolicy: {{ .Values.deployment.image.pullPolicy }} ports: - containerPort: {{ .Values.deployment.config.port }} + env: + - name: "BAL_CONFIG_FILES" + value: "/home/consolidator/conf/Config.toml" volumeMounts: - name: "config-config-map-volume" - mountPath: "/home/ballerina" + mountPath: "/home/consolidator/conf" readOnly: true {{- range .Values.deployment.secrets }} - name: {{ .name }} mountPath: {{ .mountPath }} - subPath: {{ .subPath }} + # subPath: {{ .subPath }} readOnly: true {{- end }} @@ -55,4 +58,7 @@ spec: - name: {{ .name }} secret: secretName: {{ .name }} + # items: + # - key: {{ .fileKey }} + # path: {{ .subPath }} {{- end }} diff --git a/examples/kafka-hub/helm/consolidator/templates/secret.yaml b/examples/kafka-hub/helm/consolidator/templates/secret.yaml index 00f1d909..6c0f1266 100644 --- a/examples/kafka-hub/helm/consolidator/templates/secret.yaml +++ b/examples/kafka-hub/helm/consolidator/templates/secret.yaml @@ -21,6 +21,8 @@ metadata: name: {{ .name }} type: Opaque data: - {{ .subPath }}: {{ $.Files.Get .filePath | b64enc }} + {{- range .content }} + {{ .fileKey }}: {{ $.Files.Get .filePath | b64enc }} + {{- end }} --- {{- end }} diff --git a/examples/kafka-hub/helm/consolidator/values.yaml b/examples/kafka-hub/helm/consolidator/values.yaml index e3945ec2..ff914aa0 100644 --- a/examples/kafka-hub/helm/consolidator/values.yaml +++ b/examples/kafka-hub/helm/consolidator/values.yaml @@ -17,18 +17,17 @@ deployment: replicas: 1 image: - repository: "ayeshalmeida/consolidator" + repository: "ballerina/consolidator" pullPolicy: IfNotPresent tag: 7.0.0 secrets: - - name: ballerina-consolidator-broker-truststore - mountPath: "/home/ballerina/resources/brokercerts/client-truststore.jks" - subPath: "client-truststore.jks" - filePath: "_resources/kafka-client/kafka-client.trustStore.jks" - - name: ballerina-consolidator-broker-keystore - mountPath: "/home/ballerina/resources/brokercerts/client-keystore.jks" - subPath: "client-keystore.jks" - filePath: "_resources/kafka-client/kafka-client.keystore.jks" + - name: ballerina-consolidator-secrets + mountPath: "/home/consolidator/resources/brokercerts" + content: + - filePath: "_resources/kafka-client/kafka-client.trustStore.jks" + fileKey: "client-truststore.jks" + - filePath: "_resources/kafka-client/kafka-client.keystore.jks" + fileKey: "client-keystore.jks" config: port: 10001 kafka: From ee2a8b77a5a30472c3de700091e143737d7bf6ad Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 00:10:17 +0530 Subject: [PATCH 24/39] Remove unwanted comments --- .../kafka-hub/helm/consolidator/templates/deployment.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/kafka-hub/helm/consolidator/templates/deployment.yaml b/examples/kafka-hub/helm/consolidator/templates/deployment.yaml index cf74ae56..8af6cb2e 100644 --- a/examples/kafka-hub/helm/consolidator/templates/deployment.yaml +++ b/examples/kafka-hub/helm/consolidator/templates/deployment.yaml @@ -46,7 +46,6 @@ spec: {{- range .Values.deployment.secrets }} - name: {{ .name }} mountPath: {{ .mountPath }} - # subPath: {{ .subPath }} readOnly: true {{- end }} @@ -58,7 +57,4 @@ spec: - name: {{ .name }} secret: secretName: {{ .name }} - # items: - # - key: {{ .fileKey }} - # path: {{ .subPath }} {{- end }} From 005519082c6c9e2c3a40647c0fe6e74b2372f2c9 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 00:25:58 +0530 Subject: [PATCH 25/39] Fix config issues in the hub helm-charts --- examples/kafka-hub/helm/hub/confs/Config.toml | 13 +++++-- .../kafka-hub/helm/hub/templates/conf.yaml | 2 +- .../kafka-hub/helm/hub/templates/secret.yaml | 4 +- .../helm/hub/templates/statefulset.yaml | 5 ++- examples/kafka-hub/helm/hub/values.yaml | 37 ++++++++++--------- 5 files changed, 37 insertions(+), 24 deletions(-) diff --git a/examples/kafka-hub/helm/hub/confs/Config.toml b/examples/kafka-hub/helm/hub/confs/Config.toml index 03e1847b..225aaea9 100644 --- a/examples/kafka-hub/helm/hub/confs/Config.toml +++ b/examples/kafka-hub/helm/hub/confs/Config.toml @@ -43,6 +43,13 @@ GRACEFUL_CLOSE_PERIOD = 5.0 # The port that is used to start the hub HUB_PORT = {{ .Values.deployment.config.port }} +# SSL keystore file path +# SSL_KEYSTORE_PATH = "./resources/hub.keystore.jks" +SSL_KEYSTORE_PATH = "/home/websubhub/resources/ssl/{{ .Values.deployment.config.ssl.keystore_name }}" + +# SSL keystore password +KEYSTORE_PASSWORD = {{ .Values.deployment.config.ssl.keystore_password | quote }} + # The period between retry requests MESSAGE_DELIVERY_RETRY_INTERVAL = 3.0 @@ -60,14 +67,14 @@ MESSAGE_DELIVERY_RETRYABLE_STATUS_CODES = [{{ join ", " .Values.deployment.confi issuer = {{ .Values.deployment.config.idp.jwt_issuer | quote }} audience = {{ .Values.deployment.config.idp.jwt_audience | quote }} jwksUrl = {{ .Values.deployment.config.idp.jwt_jwks_endpoint | quote }} -trustStore = "/home/ballerina/resources/{{ .Values.deployment.config.idp.truststore_name }}" +trustStore = "/home/websubhub/resources/idp/{{ .Values.deployment.config.idp.truststore_name }}" trustStorePassword = {{ .Values.deployment.config.idp.truststore_password | quote }} # The MTLS configurations related to Kafka connection [kafkaHub.config.KAFKA_MTLS_CONFIG] -cert.path = "/home/ballerina/resources/brokercerts/{{ .Values.deployment.config.kafka.truststore_name }}" +cert.path = "/home/websubhub/resources/brokercerts/{{ .Values.deployment.config.kafka.truststore_name }}" cert.password = {{ .Values.deployment.config.kafka.truststore_password | quote }} -key.keyStore.path = "/home/ballerina/resources/brokercerts/{{ .Values.deployment.config.kafka.keystore_name }}" +key.keyStore.path = "/home/websubhub/resources/brokercerts/{{ .Values.deployment.config.kafka.keystore_name }}" key.keyStore.password = {{ .Values.deployment.config.kafka.keystore_password | quote }} [ballerina.log] diff --git a/examples/kafka-hub/helm/hub/templates/conf.yaml b/examples/kafka-hub/helm/hub/templates/conf.yaml index f45fafcb..d562a76e 100644 --- a/examples/kafka-hub/helm/hub/templates/conf.yaml +++ b/examples/kafka-hub/helm/hub/templates/conf.yaml @@ -20,4 +20,4 @@ metadata: name: websubhub-svc-cm namespace : {{ .Release.Namespace }} data: - config.toml: {{ tpl (.Files.Get "confs/Config.toml") . | quote }} + Config.toml: {{ tpl (.Files.Get "confs/Config.toml") . | quote }} diff --git a/examples/kafka-hub/helm/hub/templates/secret.yaml b/examples/kafka-hub/helm/hub/templates/secret.yaml index 00f1d909..6c0f1266 100644 --- a/examples/kafka-hub/helm/hub/templates/secret.yaml +++ b/examples/kafka-hub/helm/hub/templates/secret.yaml @@ -21,6 +21,8 @@ metadata: name: {{ .name }} type: Opaque data: - {{ .subPath }}: {{ $.Files.Get .filePath | b64enc }} + {{- range .content }} + {{ .fileKey }}: {{ $.Files.Get .filePath | b64enc }} + {{- end }} --- {{- end }} diff --git a/examples/kafka-hub/helm/hub/templates/statefulset.yaml b/examples/kafka-hub/helm/hub/templates/statefulset.yaml index c9693299..43a27fa5 100644 --- a/examples/kafka-hub/helm/hub/templates/statefulset.yaml +++ b/examples/kafka-hub/helm/hub/templates/statefulset.yaml @@ -41,16 +41,17 @@ spec: valueFrom: fieldRef: fieldPath: {{ .Values.deployment.config.server_id }} + - name: "BAL_CONFIG_FILES" + value: "/home/websubhub/conf/Config.toml" - name: "BALLERINA_MAX_POOL_SIZE" value: "100" volumeMounts: - name: "config-config-map-volume" - mountPath: "/home/ballerina" + mountPath: "/home/websubhub/conf" readOnly: true {{- range .Values.deployment.secrets }} - name: {{ .name }} mountPath: {{ .mountPath }} - subPath: {{ .subPath }} readOnly: true {{- end }} diff --git a/examples/kafka-hub/helm/hub/values.yaml b/examples/kafka-hub/helm/hub/values.yaml index a42c6ea4..83f52adb 100644 --- a/examples/kafka-hub/helm/hub/values.yaml +++ b/examples/kafka-hub/helm/hub/values.yaml @@ -21,29 +21,32 @@ deployment: pullPolicy: IfNotPresent tag: 12.0.0 secrets: - - name: ballerina-websubhub-hub-keystore - mountPath: "/home/ballerina/resources/hub.keystore.jks" - subPath: "hub.keystore.jks" - filePath: "_resources/hub-1.keystore.jks" - - name: ballerina-websubhub-idp-truststore - mountPath: "/home/ballerina/resources/client-truststore.jks" - subPath: "client-truststore.jks" - filePath: "_resources/idp/client-truststore.jks" - - name: ballerina-websubhub-broker-truststore - mountPath: "/home/ballerina/resources/brokercerts/client-truststore.jks" - subPath: "client-truststore.jks" - filePath: "_resources/kafka-client/kafka-client.trustStore.jks" - - name: ballerina-websubhub-broker-keystore - mountPath: "/home/ballerina/resources/brokercerts/client-truststore.jks" - subPath: "client-keystore.jks" - filePath: "_resources/kafka-client/kafka-client.keystore.jks" + - name: ballerina-websubhub-ssl + mountPath: "/home/websubhub/resources/ssl" + content: + - filePath: "_resources/hub-1.keystore.jks" + fileKey: "hub.keystore.jks" + - name: ballerina-websubhub-idp + mountPath: "/home/websubhub/resources/idp" + content: + - filePath: "_resources/idp/client-truststore.jks" + fileKey: "client-truststore.jks" + - name: ballerina-consolidator-broker + mountPath: "/home/websubhub/resources/brokercerts" + content: + - filePath: "_resources/kafka-client/kafka-client.trustStore.jks" + fileKey: "client-truststore.jks" + - filePath: "_resources/kafka-client/kafka-client.keystore.jks" + fileKey: "client-keystore.jks" config: port: 9000 server_id: "hub-1" state_snapshot_endpoint: "http://ballerina-consolidator-deployment:10001" retryable_status_codes: [500, 502, 503] logLevel: "ERROR" - ssl_keystoreName: "hub.keystore.jks" + ssl: + keystore_name: "hub.keystore.jks" + keystore_password: "password" idp: jwt_issuer: "https://localhost:9443/oauth2/token" jwt_audience: "ballerina" From 8c6dfc009adfff2ad2354bbd86ff008262f6642c Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 10:54:49 +0530 Subject: [PATCH 26/39] Fix configuration name related to volume-mounts --- examples/kafka-hub/helm/hub/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kafka-hub/helm/hub/values.yaml b/examples/kafka-hub/helm/hub/values.yaml index 83f52adb..0eccd996 100644 --- a/examples/kafka-hub/helm/hub/values.yaml +++ b/examples/kafka-hub/helm/hub/values.yaml @@ -31,7 +31,7 @@ deployment: content: - filePath: "_resources/idp/client-truststore.jks" fileKey: "client-truststore.jks" - - name: ballerina-consolidator-broker + - name: ballerina-websubhub-broker mountPath: "/home/websubhub/resources/brokercerts" content: - filePath: "_resources/kafka-client/kafka-client.trustStore.jks" From 228def115a30913e72667a8703e6d9805196aef4 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 11:14:23 +0530 Subject: [PATCH 27/39] Fix configuration name related to volume-mounts --- examples/kafka-hub/helm/consolidator/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kafka-hub/helm/consolidator/values.yaml b/examples/kafka-hub/helm/consolidator/values.yaml index ff914aa0..a823acd5 100644 --- a/examples/kafka-hub/helm/consolidator/values.yaml +++ b/examples/kafka-hub/helm/consolidator/values.yaml @@ -21,7 +21,7 @@ deployment: pullPolicy: IfNotPresent tag: 7.0.0 secrets: - - name: ballerina-consolidator-secrets + - name: ballerina-consolidator-broker mountPath: "/home/consolidator/resources/brokercerts" content: - filePath: "_resources/kafka-client/kafka-client.trustStore.jks" From ea57cb9d53a544c9408cbb2a60e6ed77370c6088 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 11:14:38 +0530 Subject: [PATCH 28/39] Add a readme for the helm charts --- examples/kafka-hub/helm/README.md | 150 ++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 examples/kafka-hub/helm/README.md diff --git a/examples/kafka-hub/helm/README.md b/examples/kafka-hub/helm/README.md new file mode 100644 index 00000000..e741148f --- /dev/null +++ b/examples/kafka-hub/helm/README.md @@ -0,0 +1,150 @@ +# [KafkaHub] Helm charts + +Ballerina Kafka Websubhub is a WebSub compliant `hub` implementation backed by Apache Kafka message broker. + +[Overview of Kafka Websubhub](https://github.com/ballerina-platform/module-ballerina-websubhub/blob/kafkahub-mtls/examples/kafka-hub/A%20Guide%20on%20implementing%20Websub%20Hub%20backed%20by%20Kafka%20Message%20Broker.md) + +## Introduction + +These charts bootstrap a [Kafka Websubhub](https://github.com/ballerina-platform/module-ballerina-websubhub/blob/kafkahub-mtls/examples/kafka-hub) deployment +on a [Kubernetes](https://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +The **Kafka Websubhub** setup contains two separate components: +- **`hub`** : A WebSub compliant `hub` implementation +- **`consolidator`** : A backend service which manages the overall deployment state + +## Prerequisites + +- Kubernetes 1.32+ +- Helm 3.17.0+ + +## Deployment + +### 1. Deploying the `hub` + +Helm charts related to the `hub` component can be found in the `hub` directory. + +To install the charts with the release name `ballerina-websubhub`, go into the `hub` directory and run the following command: + +```sh + $ cd hub + $ helm install ballerina-websubhub . +``` + +To uninstall/delete the `ballerina-websubhub` statefulset, run the following command: + +```sh + $ helm delete ballerina-websubhub +``` + +#### Configurations + +The Kafka Websubhub container image related configurations can be found under `deployment.image` in the `hub/values.yaml` + +| Configuration | Description | +|-------------- |-------------------------------------| +| `repository` | Image repository. | +| `pullPolicy` | Image pull policy for the deployment| +| `tag` | Image tag | + + +All the package related configurations can be found under `deployment.config` in the `hub/values.yaml`. + +Following are a list of configurations and their usage. + +| Configuration | Description | +|-----------------------------|---------------------------------------------------------------------------------| +| `port` | The port that is used to start the hub | +| `server_id` | Server ID is used to uniquely identify each server | +| `state_snapshot_endpoint` | Consolidator HTTP endpoint to retrieve the current state snapshot | +| `retryable_status_codes` | The HTTP status codes for which the client should retry during message delivery | +| `logLevel` | The package log level | +| `ssl.keystore_name` | The name of the Java keystore file used to enable HTTPS on `hub` | +| `ssl.keystore_password` | The password for the Java keystore file | +| `idp.jwt_issuer` | The `issuer` claim for OAuth2 (JWT) token | +| `idp.jwt_audience` | The `audience` claim for OAuth2 (JWT) token | +| `idp.jwt_jwks_endpoint` | The JWKS endpoint URL to verify the JWT signature | +| `idp.truststore_name` | The client truststore file name used for JWKS connection | +| `idp.truststore_password` | The client truststore password | +| `kafka.bootstrap_node` | IP and port of the Kafka bootstrap node | +| `kafka.max_poll_records` | Maximum number of records returned in a single call to consumer-poll | +| `kafka.truststore_name` | The client truststore file name used for mTLS connection from `hub` to `broker` | +| `kafka.truststore_password` | The client truststore password | +| `kafka.keystore_name` | The client keystore file name used for mTLS connection from `hub` to `broker` | +| `kafka.keystore_password` | The client keystore password | + +#### Volume mounts + +Apart from the above configurations, several volume mounts are required for the deployment. All volume mounts are Kubernetes secrets, and they are listed under `deployment.secrets` in `hub/values.yaml`. + +Each secret follows the structure below: + +- **`name`**: Name of the secret. +- **`mountPath`**: The path on the pod where the secrets should be mounted. +- **`content`**: A list of secrets to be mounted to the pod. Each item in the list contains: + - **`filePath`**: The path to the original file. + - **`fileKey`**: The key for the secret value in `hub/secret.yaml`. + +The deployment requires three sets of volume mounts, each serving a specific purpose: + +- **`ballerina-websubhub-ssl`**: Enables HTTPS for the WebSubHub HTTP endpoint. +- **`ballerina-websubhub-idp`**: Supports secure HTTPS communication between WebSubHub and the IdP JWKS endpoint. +- **`ballerina-websubhub-broker`**: Establishes an mTLS connection between WebSubHub and the Kafka broker. + +### 2. Deploying the `consolidator` + +Helm charts related to the `consolidator` component can be found in the `consolidator` directory. + +To install the charts with the release name `ballerina-consolidator`, go into the `consolidator` directory and run the following command: + +```sh + $ cd consolidator + $ helm install ballerina-consolidator . +``` + +To uninstall/delete the `ballerina-consolidator` deployment, run the following command: + +```sh + $ helm delete ballerina-consolidator +``` + +#### Configurations + +The consolidator container image related configurations can be found under `deployment.image` in the `consolidator/values.yaml` + +| Configuration | Description | +|-------------- |-------------------------------------| +| `repository` | Image repository. | +| `pullPolicy` | Image pull policy for the deployment| +| `tag` | Image tag | + +All the package related configurations can be found under `deployment.config` in the `consolidator/values.yaml`. + +Following are a list of configurations and their usage. + +| Configuration | Description | +|-----------------------------|-----------------------------------------------------------------------------------------| +| `port` | The port that is used to start the consolidator state snapshot endpoint | +| `kafka.bootstrap_node` | IP and port of the Kafka bootstrap node | +| `kafka.max_poll_records` | Maximum number of records returned in a single call to consumer-poll | +| `kafka.truststore_name` | The client truststore file name used for mTLS connection from `consolidator` to `broker`| +| `kafka.truststore_password` | The client truststore password | +| `kafka.keystore_name` | The client keystore file name used for mTLS connection from `consolidator` to `broker` | +| `kafka.keystore_password` | The client keystore password | + + +#### Volume mounts + +Apart from the above configurations, several volume mounts are required for the deployment. All volume mounts are Kubernetes secrets, and they are listed under `deployment.secrets` in `consolidator/values.yaml`. + +Each secret follows the structure below: + +- **`name`**: Name of the secret. +- **`mountPath`**: The path on the pod where the secrets should be mounted. +- **`content`**: A list of secrets to be mounted to the pod. Each item in the list contains: + - **`filePath`**: The path to the original file. + - **`fileKey`**: The key for the secret value in `hub/secret.yaml`. + +The deployment requires three sets of volume mounts, each serving a specific purpose: + +- **`ballerina-consolidator-broker`**: Establishes an mTLS connection between consolidator and the Kafka broker. From f91dd374f375df6824d31e1b3e253a6c20d8ba46 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 11:15:33 +0530 Subject: [PATCH 29/39] Update Readme heading name --- examples/kafka-hub/helm/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kafka-hub/helm/README.md b/examples/kafka-hub/helm/README.md index e741148f..00aa61d8 100644 --- a/examples/kafka-hub/helm/README.md +++ b/examples/kafka-hub/helm/README.md @@ -1,4 +1,4 @@ -# [KafkaHub] Helm charts +# [Kafka Websubhub] Helm charts Ballerina Kafka Websubhub is a WebSub compliant `hub` implementation backed by Apache Kafka message broker. From 9ed6f5e25aaf6f3a407b1c6f6af6f561632789c8 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 11:18:18 +0530 Subject: [PATCH 30/39] Refactor the readme content --- examples/kafka-hub/helm/README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/kafka-hub/helm/README.md b/examples/kafka-hub/helm/README.md index 00aa61d8..51a9c96e 100644 --- a/examples/kafka-hub/helm/README.md +++ b/examples/kafka-hub/helm/README.md @@ -39,7 +39,7 @@ To uninstall/delete the `ballerina-websubhub` statefulset, run the following com #### Configurations -The Kafka Websubhub container image related configurations can be found under `deployment.image` in the `hub/values.yaml` +The Kafka Websubhub container image related configurations can be found under `deployment.image` in the `hub/values.yaml`. | Configuration | Description | |-------------- |-------------------------------------| @@ -79,17 +79,17 @@ Apart from the above configurations, several volume mounts are required for the Each secret follows the structure below: -- **`name`**: Name of the secret. -- **`mountPath`**: The path on the pod where the secrets should be mounted. +- **`name`**: Name of the secret +- **`mountPath`**: The path on the pod where the secrets should be mounted - **`content`**: A list of secrets to be mounted to the pod. Each item in the list contains: - - **`filePath`**: The path to the original file. - - **`fileKey`**: The key for the secret value in `hub/secret.yaml`. + - **`filePath`**: The path to the original file + - **`fileKey`**: The key for the secret value in `hub/secret.yaml` The deployment requires three sets of volume mounts, each serving a specific purpose: -- **`ballerina-websubhub-ssl`**: Enables HTTPS for the WebSubHub HTTP endpoint. -- **`ballerina-websubhub-idp`**: Supports secure HTTPS communication between WebSubHub and the IdP JWKS endpoint. -- **`ballerina-websubhub-broker`**: Establishes an mTLS connection between WebSubHub and the Kafka broker. +- **`ballerina-websubhub-ssl`**: Enables HTTPS for the WebSubHub HTTP endpoint +- **`ballerina-websubhub-idp`**: Supports secure HTTPS communication between WebSubHub and the IdP JWKS endpoint +- **`ballerina-websubhub-broker`**: Establishes an mTLS connection between WebSubHub and the Kafka broker ### 2. Deploying the `consolidator` @@ -110,7 +110,7 @@ To uninstall/delete the `ballerina-consolidator` deployment, run the following c #### Configurations -The consolidator container image related configurations can be found under `deployment.image` in the `consolidator/values.yaml` +The consolidator container image related configurations can be found under `deployment.image` in the `consolidator/values.yaml`. | Configuration | Description | |-------------- |-------------------------------------| @@ -139,12 +139,12 @@ Apart from the above configurations, several volume mounts are required for the Each secret follows the structure below: -- **`name`**: Name of the secret. -- **`mountPath`**: The path on the pod where the secrets should be mounted. +- **`name`**: Name of the secret +- **`mountPath`**: The path on the pod where the secrets should be mounted - **`content`**: A list of secrets to be mounted to the pod. Each item in the list contains: - - **`filePath`**: The path to the original file. - - **`fileKey`**: The key for the secret value in `hub/secret.yaml`. + - **`filePath`**: The path to the original file + - **`fileKey`**: The key for the secret value in `hub/secret.yaml` The deployment requires three sets of volume mounts, each serving a specific purpose: -- **`ballerina-consolidator-broker`**: Establishes an mTLS connection between consolidator and the Kafka broker. +- **`ballerina-consolidator-broker`**: Establishes an mTLS connection between consolidator and the Kafka broker From 8bff99d924bf7d31f12e1a8a3f1bc87581b5c502 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 12:50:01 +0530 Subject: [PATCH 31/39] Update hub and consolidator image versions --- examples/kafka-hub/consolidator/Cloud.toml | 2 +- examples/kafka-hub/docker-compose.yaml | 4 ++-- examples/kafka-hub/helm/consolidator/values.yaml | 2 +- examples/kafka-hub/helm/hub/values.yaml | 2 +- examples/kafka-hub/hub/Cloud.toml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/kafka-hub/consolidator/Cloud.toml b/examples/kafka-hub/consolidator/Cloud.toml index 6f0e1abf..aa31bcd4 100644 --- a/examples/kafka-hub/consolidator/Cloud.toml +++ b/examples/kafka-hub/consolidator/Cloud.toml @@ -1,7 +1,7 @@ [container.image] repository="ballerina" name="consolidator" -tag="7.0.0" +tag="8.0.0" [[container.copy.files]] sourceFile="./resources" diff --git a/examples/kafka-hub/docker-compose.yaml b/examples/kafka-hub/docker-compose.yaml index 238df995..e6f2be0c 100644 --- a/examples/kafka-hub/docker-compose.yaml +++ b/examples/kafka-hub/docker-compose.yaml @@ -3,7 +3,7 @@ name: 'kafkahub' services: hub-1: - image: 'ayeshalmeida/kafkahub:12.0.0' + image: 'ayeshalmeida/kafkahub:13.0.0' hostname: hub1 container_name: hub-1 ports: @@ -59,7 +59,7 @@ services: - hub_network consolidator: - image: 'ayeshalmeida/consolidator:7.0.0' + image: 'ayeshalmeida/consolidator:8.0.0' hostname: consolidator container_name: consolidator ports: diff --git a/examples/kafka-hub/helm/consolidator/values.yaml b/examples/kafka-hub/helm/consolidator/values.yaml index a823acd5..3a5649d4 100644 --- a/examples/kafka-hub/helm/consolidator/values.yaml +++ b/examples/kafka-hub/helm/consolidator/values.yaml @@ -19,7 +19,7 @@ deployment: image: repository: "ballerina/consolidator" pullPolicy: IfNotPresent - tag: 7.0.0 + tag: 8.0.0 secrets: - name: ballerina-consolidator-broker mountPath: "/home/consolidator/resources/brokercerts" diff --git a/examples/kafka-hub/helm/hub/values.yaml b/examples/kafka-hub/helm/hub/values.yaml index 0eccd996..4206eda9 100644 --- a/examples/kafka-hub/helm/hub/values.yaml +++ b/examples/kafka-hub/helm/hub/values.yaml @@ -19,7 +19,7 @@ deployment: image: repository: "ayeshalmeida/kafkahub" pullPolicy: IfNotPresent - tag: 12.0.0 + tag: 13.0.0 secrets: - name: ballerina-websubhub-ssl mountPath: "/home/websubhub/resources/ssl" diff --git a/examples/kafka-hub/hub/Cloud.toml b/examples/kafka-hub/hub/Cloud.toml index fe518af5..6418beaf 100644 --- a/examples/kafka-hub/hub/Cloud.toml +++ b/examples/kafka-hub/hub/Cloud.toml @@ -1,7 +1,7 @@ [container.image] repository="ballerina" name="kafkahub" -tag="12.0.0" +tag="13.0.0" [[container.copy.files]] sourceFile="./resources" From 52d6d46b5a532702bcd14b275bcae7f879aa7ed9 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 13:32:44 +0530 Subject: [PATCH 32/39] Update username in the docker container --- examples/kafka-hub/consolidator/Dockerfile | 2 +- examples/kafka-hub/hub/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/kafka-hub/consolidator/Dockerfile b/examples/kafka-hub/consolidator/Dockerfile index 940db40c..1c78bab2 100644 --- a/examples/kafka-hub/consolidator/Dockerfile +++ b/examples/kafka-hub/consolidator/Dockerfile @@ -24,7 +24,7 @@ LABEL maintainer="WSO2 Docker Maintainers " # set Docker image build arguments # build arguments for user/group configurations -ARG USER=consolidator +ARG USER=ballerina ARG USER_ID=10001 ARG USER_GROUP=wso2 ARG USER_GROUP_ID=10001 diff --git a/examples/kafka-hub/hub/Dockerfile b/examples/kafka-hub/hub/Dockerfile index f94d6854..d4f2a48b 100644 --- a/examples/kafka-hub/hub/Dockerfile +++ b/examples/kafka-hub/hub/Dockerfile @@ -24,7 +24,7 @@ LABEL maintainer="WSO2 Docker Maintainers " # set Docker image build arguments # build arguments for user/group configurations -ARG USER=websubhub +ARG USER=ballerina ARG USER_ID=10001 ARG USER_GROUP=wso2 ARG USER_GROUP_ID=10001 From eab7239f9984deff845c31a91de1bf6bb98f8a41 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 14:04:41 +0530 Subject: [PATCH 33/39] Fix consolidator container configurations in the docker-compose --- .../_resources/consolidator.config.toml | 25 +++++++++++++++++++ examples/kafka-hub/docker-compose.yaml | 4 ++- 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 examples/kafka-hub/_resources/consolidator.config.toml diff --git a/examples/kafka-hub/_resources/consolidator.config.toml b/examples/kafka-hub/_resources/consolidator.config.toml new file mode 100644 index 00000000..dcbccb86 --- /dev/null +++ b/examples/kafka-hub/_resources/consolidator.config.toml @@ -0,0 +1,25 @@ +[consolidatorService.config] +# IP and Port of the Kafka bootstrap node +KAFKA_BOOTSTRAP_NODE = "localhost:9094" + +# Kafka topic which stores websub-events for this server +WEBSUB_EVENTS_TOPIC = "websub-events" + +# Kafka topic which stores the current snapshot for the websub-events +WEBSUB_EVENTS_SNAPSHOT_TOPIC = "websub-events-snapshot" + +# The interval in which Kafka consumers wait for new messages +POLLING_INTERVAL = 10.0 + +# The period in which Kafka close method waits to complete +GRACEFUL_CLOSE_PERIOD = 5.0 + +# The port that is used to start the HTTP endpoint for consolidator +CONSOLIDATOR_HTTP_ENDPOINT_PORT = 10001 + +# The MTLS configurations related to Kafka connection +[consolidatorService.config.KAFKA_MTLS_CONFIG] +cert.path = "./resources/brokercerts/client-truststore.jks" +cert.password = "password" +key.keyStore.path = "./resources/brokercerts/client-keystore.jks" +key.keyStore.password = "password" diff --git a/examples/kafka-hub/docker-compose.yaml b/examples/kafka-hub/docker-compose.yaml index e6f2be0c..4f1dfd52 100644 --- a/examples/kafka-hub/docker-compose.yaml +++ b/examples/kafka-hub/docker-compose.yaml @@ -83,12 +83,14 @@ services: broker: condition: service_healthy volumes: + # `Config.toml` file for the consolidator-service + - ./_resources/consolidator.config.toml:/home/ballerina/Config.toml # Kafka client truststore file - ./_resources/secrets/kafka-client/kafka-client.trustStore.jks:/home/ballerina/resources/brokercerts/client-truststore.jks # Kafka client keystore file - ./_resources/secrets/kafka-client/kafka-client.keystore.jks:/home/ballerina/resources/brokercerts/client-keystore.jks healthcheck: - test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider 'http://consolidator:10001/health/readiness' || exit 1"] + test: ["CMD-SHELL", "curl --fail http://consolidator:10001/health/readiness || exit 1"] interval: 30s timeout: 10s start_period: 30s From 2a1229330ca8e05ea9565320262b4b374a9a1a46 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 14:28:02 +0530 Subject: [PATCH 34/39] Fix hub container configurations in the docker-compose --- examples/kafka-hub/_resources/hub.config.toml | 64 +++++++++++++++++++ examples/kafka-hub/docker-compose.yaml | 4 +- 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 examples/kafka-hub/_resources/hub.config.toml diff --git a/examples/kafka-hub/_resources/hub.config.toml b/examples/kafka-hub/_resources/hub.config.toml new file mode 100644 index 00000000..e0642102 --- /dev/null +++ b/examples/kafka-hub/_resources/hub.config.toml @@ -0,0 +1,64 @@ +[kafkaHub.config] +# Flag to check whether to enable/disable security +SECURITY_ON = true + +# Server ID is is used to uniquely identify each server +# Each server must have a unique ID +SERVER_ID = "hub-1" + +# IP and Port of the Kafka bootstrap node +KAFKA_BOOTSTRAP_NODE = "localhost:9094" + +# Maximum number of records returned in a single call to consumer-poll +KAFKA_CONSUMER_MAX_POLL_RECORDS = 50 + +# Kafka topic which is stores websub-events for this server +WEBSUB_EVENTS_TOPIC = "websub-events" + +# Consolidator HTTP endpoint to be used to retrieve current state-snapshot +STATE_SNAPSHOT_ENDPOINT = "http://localhost:10001" + +# The interval in which Kafka consumers wait for new messages +POLLING_INTERVAL = 10.0 + +# The period in which Kafka close method waits to complete +GRACEFUL_CLOSE_PERIOD = 5.0 + +# The port that is used to start the hub +HUB_PORT = 9000 + +# SSL keystore file path +SSL_KEYSTORE_PATH = "./resources/hub.keystore.jks" + +# SSL keystore password +KEYSTORE_PASSWORD = "password" + +# The period between retry requests +MESSAGE_DELIVERY_RETRY_INTERVAL = 3.0 + +# The maximum retry count +MESSAGE_DELIVERY_COUNT = 3 + +# The message delivery timeout +MESSAGE_DELIVERY_TIMEOUT = 10.0 + +# The HTTP status codes for which the client should retry +MESSAGE_DELIVERY_RETRYABLE_STATUS_CODES = [500, 502, 503] + +# The Oauth2 authorization related configurations +[kafkaHub.config.OAUTH2_CONFIG] +issuer = "https://localhost:9443/oauth2/token" +audience = "ballerina" +jwksUrl = "https://localhost:9443/oauth2/jwks" +trustStore = "./resources/client-truststore.jks" +trustStorePassword = "wso2carbon" + +# The MTLS configurations related to Kafka connection +[kafkaHub.config.KAFKA_MTLS_CONFIG] +cert.path = "./resources/brokercerts/client-truststore.jks" +cert.password = "password" +key.keyStore.path = "./resources/brokercerts/client-keystore.jks" +key.keyStore.password = "password" + +[ballerina.log] +level = "DEBUG" diff --git a/examples/kafka-hub/docker-compose.yaml b/examples/kafka-hub/docker-compose.yaml index 4f1dfd52..19b23318 100644 --- a/examples/kafka-hub/docker-compose.yaml +++ b/examples/kafka-hub/docker-compose.yaml @@ -41,6 +41,8 @@ services: # The HTTP status codes for which the client should retry RETRYABLE_STATUS_CODES: "500,502,503" volumes: + # `Config.toml` file for the websubhub-service + - ./_resources/hub.config.toml:/home/ballerina/Config.toml # Kafka client truststore file - ./_resources/secrets/kafka-client/kafka-client.trustStore.jks:/home/ballerina/resources/brokercerts/client-truststore.jks # Kafka client keystore file @@ -50,7 +52,7 @@ services: # SSL configurations - ./_resources/secrets/hub/hub-1.keystore.jks:/home/ballerina/resources/hub.keystore.jks healthcheck: - test: ["CMD-SHELL", "wget --no-verbose --no-check-certificate --tries=1 --spider 'https://hub1:9000/health' || exit 1"] + test: ["CMD-SHELL", "curl -k --fail https://hub1:9000/health || exit 1"] interval: 30s timeout: 10s start_period: 30s From 24e3fa2a5b55fa657d644f3fe6ee27688d25359a Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 14:35:15 +0530 Subject: [PATCH 35/39] Fix paths in the hub configurations --- examples/kafka-hub/helm/hub/confs/Config.toml | 8 ++++---- examples/kafka-hub/helm/hub/templates/statefulset.yaml | 4 ++-- examples/kafka-hub/helm/hub/values.yaml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/kafka-hub/helm/hub/confs/Config.toml b/examples/kafka-hub/helm/hub/confs/Config.toml index 225aaea9..60a4ce22 100644 --- a/examples/kafka-hub/helm/hub/confs/Config.toml +++ b/examples/kafka-hub/helm/hub/confs/Config.toml @@ -45,7 +45,7 @@ HUB_PORT = {{ .Values.deployment.config.port }} # SSL keystore file path # SSL_KEYSTORE_PATH = "./resources/hub.keystore.jks" -SSL_KEYSTORE_PATH = "/home/websubhub/resources/ssl/{{ .Values.deployment.config.ssl.keystore_name }}" +SSL_KEYSTORE_PATH = "/home/ballerina/resources/ssl/{{ .Values.deployment.config.ssl.keystore_name }}" # SSL keystore password KEYSTORE_PASSWORD = {{ .Values.deployment.config.ssl.keystore_password | quote }} @@ -67,14 +67,14 @@ MESSAGE_DELIVERY_RETRYABLE_STATUS_CODES = [{{ join ", " .Values.deployment.confi issuer = {{ .Values.deployment.config.idp.jwt_issuer | quote }} audience = {{ .Values.deployment.config.idp.jwt_audience | quote }} jwksUrl = {{ .Values.deployment.config.idp.jwt_jwks_endpoint | quote }} -trustStore = "/home/websubhub/resources/idp/{{ .Values.deployment.config.idp.truststore_name }}" +trustStore = "/home/ballerina/resources/idp/{{ .Values.deployment.config.idp.truststore_name }}" trustStorePassword = {{ .Values.deployment.config.idp.truststore_password | quote }} # The MTLS configurations related to Kafka connection [kafkaHub.config.KAFKA_MTLS_CONFIG] -cert.path = "/home/websubhub/resources/brokercerts/{{ .Values.deployment.config.kafka.truststore_name }}" +cert.path = "/home/ballerina/resources/brokercerts/{{ .Values.deployment.config.kafka.truststore_name }}" cert.password = {{ .Values.deployment.config.kafka.truststore_password | quote }} -key.keyStore.path = "/home/websubhub/resources/brokercerts/{{ .Values.deployment.config.kafka.keystore_name }}" +key.keyStore.path = "/home/ballerina/resources/brokercerts/{{ .Values.deployment.config.kafka.keystore_name }}" key.keyStore.password = {{ .Values.deployment.config.kafka.keystore_password | quote }} [ballerina.log] diff --git a/examples/kafka-hub/helm/hub/templates/statefulset.yaml b/examples/kafka-hub/helm/hub/templates/statefulset.yaml index 43a27fa5..5c43dfea 100644 --- a/examples/kafka-hub/helm/hub/templates/statefulset.yaml +++ b/examples/kafka-hub/helm/hub/templates/statefulset.yaml @@ -42,12 +42,12 @@ spec: fieldRef: fieldPath: {{ .Values.deployment.config.server_id }} - name: "BAL_CONFIG_FILES" - value: "/home/websubhub/conf/Config.toml" + value: "/home/ballerina/conf/Config.toml" - name: "BALLERINA_MAX_POOL_SIZE" value: "100" volumeMounts: - name: "config-config-map-volume" - mountPath: "/home/websubhub/conf" + mountPath: "/home/ballerina/conf" readOnly: true {{- range .Values.deployment.secrets }} - name: {{ .name }} diff --git a/examples/kafka-hub/helm/hub/values.yaml b/examples/kafka-hub/helm/hub/values.yaml index 4206eda9..5f9466fa 100644 --- a/examples/kafka-hub/helm/hub/values.yaml +++ b/examples/kafka-hub/helm/hub/values.yaml @@ -22,17 +22,17 @@ deployment: tag: 13.0.0 secrets: - name: ballerina-websubhub-ssl - mountPath: "/home/websubhub/resources/ssl" + mountPath: "/home/ballerina/resources/ssl" content: - filePath: "_resources/hub-1.keystore.jks" fileKey: "hub.keystore.jks" - name: ballerina-websubhub-idp - mountPath: "/home/websubhub/resources/idp" + mountPath: "/home/ballerina/resources/idp" content: - filePath: "_resources/idp/client-truststore.jks" fileKey: "client-truststore.jks" - name: ballerina-websubhub-broker - mountPath: "/home/websubhub/resources/brokercerts" + mountPath: "/home/ballerina/resources/brokercerts" content: - filePath: "_resources/kafka-client/kafka-client.trustStore.jks" fileKey: "client-truststore.jks" From 96a56049bf2cfbf5038ae432e034504d00a05c25 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 14:36:17 +0530 Subject: [PATCH 36/39] Fix paths in the consolidator configurations --- examples/kafka-hub/helm/consolidator/confs/Config.toml | 4 ++-- .../kafka-hub/helm/consolidator/templates/deployment.yaml | 4 ++-- examples/kafka-hub/helm/consolidator/values.yaml | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/kafka-hub/helm/consolidator/confs/Config.toml b/examples/kafka-hub/helm/consolidator/confs/Config.toml index cfafecb1..1118b3c8 100644 --- a/examples/kafka-hub/helm/consolidator/confs/Config.toml +++ b/examples/kafka-hub/helm/consolidator/confs/Config.toml @@ -35,7 +35,7 @@ CONSOLIDATOR_HTTP_ENDPOINT_PORT = {{ .Values.deployment.config.port }} # The MTLS configurations related to Kafka connection [consolidatorService.config.KAFKA_MTLS_CONFIG] -cert.path = "/home/consolidator/resources/brokercerts/{{ .Values.deployment.config.kafka.truststore_name }}" +cert.path = "/home/ballerina/resources/brokercerts/{{ .Values.deployment.config.kafka.truststore_name }}" cert.password = {{ .Values.deployment.config.kafka.truststore_password | quote }} -key.keyStore.path = "/home/consolidator/resources/brokercerts/{{ .Values.deployment.config.kafka.keystore_name }}" +key.keyStore.path = "/home/ballerina/resources/brokercerts/{{ .Values.deployment.config.kafka.keystore_name }}" key.keyStore.password = {{ .Values.deployment.config.kafka.keystore_password | quote }} diff --git a/examples/kafka-hub/helm/consolidator/templates/deployment.yaml b/examples/kafka-hub/helm/consolidator/templates/deployment.yaml index 8af6cb2e..83287225 100644 --- a/examples/kafka-hub/helm/consolidator/templates/deployment.yaml +++ b/examples/kafka-hub/helm/consolidator/templates/deployment.yaml @@ -38,10 +38,10 @@ spec: - containerPort: {{ .Values.deployment.config.port }} env: - name: "BAL_CONFIG_FILES" - value: "/home/consolidator/conf/Config.toml" + value: "/home/ballerina/conf/Config.toml" volumeMounts: - name: "config-config-map-volume" - mountPath: "/home/consolidator/conf" + mountPath: "/home/ballerina/conf" readOnly: true {{- range .Values.deployment.secrets }} - name: {{ .name }} diff --git a/examples/kafka-hub/helm/consolidator/values.yaml b/examples/kafka-hub/helm/consolidator/values.yaml index 3a5649d4..94b9c431 100644 --- a/examples/kafka-hub/helm/consolidator/values.yaml +++ b/examples/kafka-hub/helm/consolidator/values.yaml @@ -17,12 +17,12 @@ deployment: replicas: 1 image: - repository: "ballerina/consolidator" + repository: "ballerina/ballerina" pullPolicy: IfNotPresent tag: 8.0.0 secrets: - name: ballerina-consolidator-broker - mountPath: "/home/consolidator/resources/brokercerts" + mountPath: "/home/ballerina/resources/brokercerts" content: - filePath: "_resources/kafka-client/kafka-client.trustStore.jks" fileKey: "client-truststore.jks" @@ -43,5 +43,5 @@ ingress: hosts: - host: ballerina-consolidator.local paths: - - path: /consolidator + - path: /ballerina pathType: Prefix From 4207f44c1028771d2f3812d1af7efaafbd0d4cbf Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 14:55:34 +0530 Subject: [PATCH 37/39] Fix runtime errors --- examples/kafka-hub/helm/consolidator/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kafka-hub/helm/consolidator/values.yaml b/examples/kafka-hub/helm/consolidator/values.yaml index 94b9c431..11b360b7 100644 --- a/examples/kafka-hub/helm/consolidator/values.yaml +++ b/examples/kafka-hub/helm/consolidator/values.yaml @@ -17,7 +17,7 @@ deployment: replicas: 1 image: - repository: "ballerina/ballerina" + repository: "ayeshalmeida/consolidator" pullPolicy: IfNotPresent tag: 8.0.0 secrets: From 69860a93843535961ec71cfbb96d6dee5b2f865b Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 15:13:47 +0530 Subject: [PATCH 38/39] Fix runtime issues --- examples/kafka-hub/helm/hub/templates/statefulset.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/kafka-hub/helm/hub/templates/statefulset.yaml b/examples/kafka-hub/helm/hub/templates/statefulset.yaml index 5c43dfea..4bef7141 100644 --- a/examples/kafka-hub/helm/hub/templates/statefulset.yaml +++ b/examples/kafka-hub/helm/hub/templates/statefulset.yaml @@ -37,10 +37,6 @@ spec: ports: - containerPort: {{ .Values.deployment.config.port }} env: - - name: "SERVER_ID" - valueFrom: - fieldRef: - fieldPath: {{ .Values.deployment.config.server_id }} - name: "BAL_CONFIG_FILES" value: "/home/ballerina/conf/Config.toml" - name: "BALLERINA_MAX_POOL_SIZE" From ccf6866aeee9e6c5977d521dea7fd5e456196fa9 Mon Sep 17 00:00:00 2001 From: Ayesh Almeida Date: Fri, 7 Feb 2025 15:15:03 +0530 Subject: [PATCH 39/39] Restructure the readme --- examples/kafka-hub/helm/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kafka-hub/helm/README.md b/examples/kafka-hub/helm/README.md index 51a9c96e..129a3167 100644 --- a/examples/kafka-hub/helm/README.md +++ b/examples/kafka-hub/helm/README.md @@ -1,4 +1,4 @@ -# [Kafka Websubhub] Helm charts +# Kafka Websubhub Helm charts Ballerina Kafka Websubhub is a WebSub compliant `hub` implementation backed by Apache Kafka message broker.