From 417e2994f1e9395cc0536723ddf78f077db1dffc Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Mon, 23 Jan 2023 14:05:09 +0000 Subject: [PATCH 1/3] fix(kafka-setup): Make topic name consistent with other images --- docker/kafka-setup/Dockerfile | 8 ++++---- docker/kafka-setup/env/docker.env | 9 +++++++-- docker/kafka-setup/kafka-setup.sh | 8 ++++---- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/docker/kafka-setup/Dockerfile b/docker/kafka-setup/Dockerfile index 2bb3adc2c82840..6396d51f97485d 100644 --- a/docker/kafka-setup/Dockerfile +++ b/docker/kafka-setup/Dockerfile @@ -51,10 +51,10 @@ ENV METADATA_AUDIT_EVENT_NAME="MetadataAuditEvent_v4" ENV METADATA_CHANGE_EVENT_NAME="MetadataChangeEvent_v4" ENV FAILED_METADATA_CHANGE_EVENT_NAME="FailedMetadataChangeEvent_v4" ENV DATAHUB_USAGE_EVENT_NAME="DataHubUsageEvent_v1" -ENV METADATA_CHANGE_LOG_VERSIONED_TOPIC="MetadataChangeLog_Versioned_v1" -ENV METADATA_CHANGE_LOG_TIMESERIES_TOPIC="MetadataChangeLog_Timeseries_v1" -ENV METADATA_CHANGE_PROPOSAL_TOPIC="MetadataChangeProposal_v1" -ENV FAILED_METADATA_CHANGE_PROPOSAL_TOPIC="FailedMetadataChangeProposal_v1" +ENV METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME="MetadataChangeLog_Versioned_v1" +ENV METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME="MetadataChangeLog_Timeseries_v1" +ENV METADATA_CHANGE_PROPOSAL_TOPIC_NAME="MetadataChangeProposal_v1" +ENV FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME="FailedMetadataChangeProposal_v1" ENV PLATFORM_EVENT_TOPIC_NAME="PlatformEvent_v1" COPY docker/kafka-setup/kafka-setup.sh ./kafka-setup.sh diff --git a/docker/kafka-setup/env/docker.env b/docker/kafka-setup/env/docker.env index eae5bbc99dbbaa..e36e82cc5d4919 100644 --- a/docker/kafka-setup/env/docker.env +++ b/docker/kafka-setup/env/docker.env @@ -3,11 +3,16 @@ KAFKA_BOOTSTRAP_SERVER=broker:29092 # Configure the topics that are created by kafka-setup # Make sure these names are consistent across the whole deployment +# METADATA_CHANGE_PROPOSAL_TOPIC_NAME=MetadataChangeProposal_v1 +# FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME=FailedMetadataChangeProposal_v1 +# METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 +# METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME=MetadataChangeLog_Timeseries_v1 +# PLATFORM_EVENT_TOPIC_NAME=PlatformEvent_v1 +# DATAHUB_USAGE_EVENT_NAME=DataHubUsageEvent_v1 +# Deprecated! # METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 # METADATA_CHANGE_EVENT_NAME=MetadataChangeEvent_v4 # FAILED_METADATA_CHANGE_EVENT_NAME=FailedMetadataChangeEvent_v4 -# PLATFORM_EVENT_TOPIC_NAME=PlatformEvent_v1 -# DATAHUB_USAGE_EVENT_NAME=DataHubUsageEvent_v1 # PARTITIONS=1 # REPLICATION_FACTOR=1 diff --git a/docker/kafka-setup/kafka-setup.sh b/docker/kafka-setup/kafka-setup.sh index 0f3e46ac66d85b..dce6600cfb7b87 100644 --- a/docker/kafka-setup/kafka-setup.sh +++ b/docker/kafka-setup/kafka-setup.sh @@ -100,12 +100,12 @@ send() { send "$METADATA_AUDIT_EVENT_NAME" "--topic $METADATA_AUDIT_EVENT_NAME" send "$METADATA_CHANGE_EVENT_NAME" "--topic $METADATA_CHANGE_EVENT_NAME" send "$FAILED_METADATA_CHANGE_EVENT_NAME" "--topic $FAILED_METADATA_CHANGE_EVENT_NAME" -send "$METADATA_CHANGE_LOG_VERSIONED_TOPIC" "--topic $METADATA_CHANGE_LOG_VERSIONED_TOPIC" +send "$METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME" "--topic $METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME" # Set retention to 90 days -send "$METADATA_CHANGE_LOG_TIMESERIES_TOPIC" "--config retention.ms=7776000000 --topic $METADATA_CHANGE_LOG_TIMESERIES_TOPIC" -send "$METADATA_CHANGE_PROPOSAL_TOPIC" "--topic $METADATA_CHANGE_PROPOSAL_TOPIC" -send "$FAILED_METADATA_CHANGE_PROPOSAL_TOPIC" "--topic $FAILED_METADATA_CHANGE_PROPOSAL_TOPIC" +send "$METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME" "--config retention.ms=7776000000 --topic $METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME" +send "$METADATA_CHANGE_PROPOSAL_TOPIC_NAME" "--topic $METADATA_CHANGE_PROPOSAL_TOPIC_NAME" +send "$FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME" "--topic $FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME" send "$PLATFORM_EVENT_TOPIC_NAME" "--topic $PLATFORM_EVENT_TOPIC_NAME" # Create topic for datahub usage event From e5a25dbfa0df1c13a50515ced9cabe4a453577ab Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Tue, 24 Jan 2023 15:21:55 +0000 Subject: [PATCH 2/3] Add breaking change documentation --- docs/how/updating-datahub.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 6570fd0f98273f..f39008c11d052b 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -6,6 +6,9 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ### Breaking Changes +- #7103 This should only impact users who have configured explicit non-default names for DataHub's Kafka topics. The environment variables used to configure Kafka topics for DataHub used in the `kafka-setup` docker image have been updated to be in-line with other DataHub components, for more info see our docs on [Configuring Kafka in DataHub +](https://datahubproject.io/docs/how/kafka-config). They have been suffixed with `_TOPIC`. This change should not affect any user who is using default Kafka names. + ### Potential Downtime ### Deprecations From 186ce267f417b58881c74ad428739bd7d4f1e1f0 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Tue, 24 Jan 2023 16:42:45 +0000 Subject: [PATCH 3/3] Update updating-datahub.md --- docs/how/updating-datahub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index f39008c11d052b..090751c4b8bab1 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -7,7 +7,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ### Breaking Changes - #7103 This should only impact users who have configured explicit non-default names for DataHub's Kafka topics. The environment variables used to configure Kafka topics for DataHub used in the `kafka-setup` docker image have been updated to be in-line with other DataHub components, for more info see our docs on [Configuring Kafka in DataHub -](https://datahubproject.io/docs/how/kafka-config). They have been suffixed with `_TOPIC`. This change should not affect any user who is using default Kafka names. +](https://datahubproject.io/docs/how/kafka-config). They have been suffixed with `_TOPIC` where as now the correct suffix is `_TOPIC_NAME`. This change should not affect any user who is using default Kafka names. ### Potential Downtime