fix(kafka-setup): Make topic name consistent with other images (datah…

…ub-project#7103)
ericyomi · Feb 8, 2023 · 92ee731 · 92ee731
1 parent 222b435
commit 92ee731
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 10 deletions.
diff --git a/docker/kafka-setup/Dockerfile b/docker/kafka-setup/Dockerfile
@@ -51,10 +51,10 @@ ENV METADATA_AUDIT_EVENT_NAME="MetadataAuditEvent_v4"
 ENV METADATA_CHANGE_EVENT_NAME="MetadataChangeEvent_v4"
 ENV FAILED_METADATA_CHANGE_EVENT_NAME="FailedMetadataChangeEvent_v4"
 ENV DATAHUB_USAGE_EVENT_NAME="DataHubUsageEvent_v1"
-ENV METADATA_CHANGE_LOG_VERSIONED_TOPIC="MetadataChangeLog_Versioned_v1"
-ENV METADATA_CHANGE_LOG_TIMESERIES_TOPIC="MetadataChangeLog_Timeseries_v1"
-ENV METADATA_CHANGE_PROPOSAL_TOPIC="MetadataChangeProposal_v1"
-ENV FAILED_METADATA_CHANGE_PROPOSAL_TOPIC="FailedMetadataChangeProposal_v1"
+ENV METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME="MetadataChangeLog_Versioned_v1"
+ENV METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME="MetadataChangeLog_Timeseries_v1"
+ENV METADATA_CHANGE_PROPOSAL_TOPIC_NAME="MetadataChangeProposal_v1"
+ENV FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME="FailedMetadataChangeProposal_v1"
 ENV PLATFORM_EVENT_TOPIC_NAME="PlatformEvent_v1"
 
 COPY docker/kafka-setup/kafka-setup.sh ./kafka-setup.sh

diff --git a/docker/kafka-setup/env/docker.env b/docker/kafka-setup/env/docker.env
@@ -3,11 +3,16 @@ KAFKA_BOOTSTRAP_SERVER=broker:29092
 
 # Configure the topics that are created by kafka-setup
 # Make sure these names are consistent across the whole deployment
+# METADATA_CHANGE_PROPOSAL_TOPIC_NAME=MetadataChangeProposal_v1
+# FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME=FailedMetadataChangeProposal_v1
+# METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
+# METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME=MetadataChangeLog_Timeseries_v1
+# PLATFORM_EVENT_TOPIC_NAME=PlatformEvent_v1
+# DATAHUB_USAGE_EVENT_NAME=DataHubUsageEvent_v1
+# Deprecated!
 # METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
 # METADATA_CHANGE_EVENT_NAME=MetadataChangeEvent_v4
 # FAILED_METADATA_CHANGE_EVENT_NAME=FailedMetadataChangeEvent_v4
-# PLATFORM_EVENT_TOPIC_NAME=PlatformEvent_v1
-# DATAHUB_USAGE_EVENT_NAME=DataHubUsageEvent_v1
 # PARTITIONS=1
 # REPLICATION_FACTOR=1
 

diff --git a/docker/kafka-setup/kafka-setup.sh b/docker/kafka-setup/kafka-setup.sh
@@ -100,12 +100,12 @@ send() {
 send "$METADATA_AUDIT_EVENT_NAME" "--topic $METADATA_AUDIT_EVENT_NAME"
 send "$METADATA_CHANGE_EVENT_NAME" "--topic $METADATA_CHANGE_EVENT_NAME"
 send "$FAILED_METADATA_CHANGE_EVENT_NAME" "--topic $FAILED_METADATA_CHANGE_EVENT_NAME"
-send "$METADATA_CHANGE_LOG_VERSIONED_TOPIC" "--topic $METADATA_CHANGE_LOG_VERSIONED_TOPIC"
+send "$METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME" "--topic $METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME"
 
 # Set retention to 90 days
-send "$METADATA_CHANGE_LOG_TIMESERIES_TOPIC" "--config retention.ms=7776000000 --topic $METADATA_CHANGE_LOG_TIMESERIES_TOPIC"
-send "$METADATA_CHANGE_PROPOSAL_TOPIC" "--topic $METADATA_CHANGE_PROPOSAL_TOPIC"
-send "$FAILED_METADATA_CHANGE_PROPOSAL_TOPIC" "--topic $FAILED_METADATA_CHANGE_PROPOSAL_TOPIC"
+send "$METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME" "--config retention.ms=7776000000 --topic $METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME"
+send "$METADATA_CHANGE_PROPOSAL_TOPIC_NAME" "--topic $METADATA_CHANGE_PROPOSAL_TOPIC_NAME"
+send "$FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME" "--topic $FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME"
 send "$PLATFORM_EVENT_TOPIC_NAME" "--topic $PLATFORM_EVENT_TOPIC_NAME"
 
 # Create topic for datahub usage event

diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
@@ -6,6 +6,9 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 ### Breaking Changes
 
+- #7103 This should only impact users who have configured explicit non-default names for DataHub's Kafka topics. The environment variables used to configure Kafka topics for DataHub used in the `kafka-setup` docker image have been updated to be in-line with other DataHub components, for more info see our docs on [Configuring Kafka in DataHub
+](https://datahubproject.io/docs/how/kafka-config). They have been suffixed with `_TOPIC` where as now the correct suffix is `_TOPIC_NAME`. This change should not affect any user who is using default Kafka names.
+
 ### Potential Downtime
 
 ### Deprecations