feat(quickstart): Remove kafka-setup as a hard deployment requirement

.github/workflows/build-and-test.yml

@@ -73,6 +73,10 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.7"
+      - name: Download YQ
+        uses: chrisdickinson/[email protected]
+        with:
+          yq-version: v4.28.2
       - name: Quickstart Compose Validation
         run: ./docker/quickstart/generate_and_compare.sh
 

docker/datahub-mce-consumer/env/docker.env

@@ -2,7 +2,7 @@ MCE_CONSUMER_ENABLED=true
 EBEAN_DATASOURCE_USERNAME=datahub
 EBEAN_DATASOURCE_PASSWORD=datahub
 EBEAN_DATASOURCE_HOST=mysql:3306
-EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
+EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
 EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
 KAFKA_BOOTSTRAP_SERVER=broker:29092
 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
@@ -15,6 +15,10 @@ JAVA_OPTS=-Xms1g -Xmx1g
 ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml
 DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
 DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
+ENTITY_SERVICE_ENABLE_RETENTION=true
+MAE_CONSUMER_ENABLED=false
+PE_CONSUMER_ENABLED=false
+UI_INGESTION_ENABLED=false
 
 # Uncomment to configure kafka topic names
 # Make sure these names are consistent across the whole deployment

docker/docker-compose-with-cassandra.yml

@@ -7,7 +7,7 @@
 version: '3.8'
 services:
   zookeeper:
-    image: confluentinc/cp-zookeeper:5.4.0
+    image: confluentinc/cp-zookeeper:7.2.2
     env_file: zookeeper/env/docker.env
     hostname: zookeeper
     container_name: zookeeper
@@ -17,7 +17,7 @@ services:
       - zkdata:/var/lib/zookeeper
 
   broker:
-    image: confluentinc/cp-kafka:5.4.0
+    image: confluentinc/cp-kafka:7.2.2
     env_file: broker/env/docker.env
     hostname: broker
     container_name: broker
@@ -29,25 +29,12 @@ services:
     volumes:
       - broker:/var/lib/kafka/data/
 
-  # This "container" is a workaround to pre-create topics
-  kafka-setup:
-    build:
-      context: kafka-setup
-    image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
-    env_file: kafka-setup/env/docker.env
-    hostname: kafka-setup
-    container_name: kafka-setup
-    depends_on:
-      - broker
-      - schema-registry
-
   schema-registry:
-    image: confluentinc/cp-schema-registry:5.4.0
+    image: confluentinc/cp-schema-registry:7.2.2
     env_file: schema-registry/env/docker.env
     hostname: schema-registry
     container_name: schema-registry
     depends_on:
-      - zookeeper
       - broker
     ports:
       - "8081:8081"

docker/docker-compose-without-neo4j.m1.yml

@@ -1,9 +1,3 @@
 services:
-  broker:
-    image: kymeric/cp-kafka:latest
   mysql:
     image: mariadb:10.5.8
-  schema-registry:
-    image: eugenetea/schema-registry-arm64:latest
-  zookeeper:
-    image: kymeric/cp-zookeeper:latest

docker/docker-compose-without-neo4j.override.yml

@@ -6,7 +6,7 @@ services:
     hostname: mysql
     image: mysql:5.7
     env_file: mysql/env/docker.env
-    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
+    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
     ports:
       - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
     volumes:

docker/docker-compose-without-neo4j.yml

@@ -7,7 +7,7 @@
 version: '3.8'
 services:
   zookeeper:
-    image: confluentinc/cp-zookeeper:5.4.0
+    image: confluentinc/cp-zookeeper:7.2.2
     env_file: zookeeper/env/docker.env
     hostname: zookeeper
     container_name: zookeeper
@@ -17,7 +17,7 @@ services:
       - zkdata:/var/lib/zookeeper
 
   broker:
-    image: confluentinc/cp-kafka:5.4.0
+    image: confluentinc/cp-kafka:7.2.2
     env_file: broker/env/docker.env
     hostname: broker
     container_name: broker
@@ -26,25 +26,12 @@ services:
     ports:
       - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
 
-  # This "container" is a workaround to pre-create topics
-  kafka-setup:
-    build:
-      context: kafka-setup
-    image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
-    env_file: kafka-setup/env/docker.env
-    hostname: kafka-setup
-    container_name: kafka-setup
-    depends_on:
-      - broker
-      - schema-registry
-
   schema-registry:
-    image: confluentinc/cp-schema-registry:5.4.0
+    image: confluentinc/cp-schema-registry:7.2.2
     env_file: schema-registry/env/docker.env
     hostname: schema-registry
     container_name: schema-registry
     depends_on:
-      - zookeeper
       - broker
     ports:
       - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081

docker/docker-compose.consumers-without-neo4j.yml

@@ -25,5 +25,8 @@ services:
     env_file: datahub-mce-consumer/env/docker.env
     hostname: datahub-mce-consumer
     container_name: datahub-mce-consumer
+    environment:
+      - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
+      - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
     ports:
       - "9090:9090"

docker/docker-compose.consumers.yml

@@ -27,5 +27,15 @@ services:
     env_file: datahub-mce-consumer/env/docker.env
     hostname: datahub-mce-consumer
     container_name: datahub-mce-consumer
+    environment:
+      - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
+      - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
+      - NEO4J_HOST=http://neo4j:7474
+      - NEO4J_URI=bolt://neo4j
+      - NEO4J_USERNAME=neo4j
+      - NEO4J_PASSWORD=datahub
+      - GRAPH_SERVICE_IMPL=neo4j
     ports:
       - "9090:9090"
+    depends_on:
+      - neo4j

docker/docker-compose.dev.yml

@@ -24,17 +24,6 @@ services:
       - ./elasticsearch-setup/create-indices.sh:/create-indices.sh
       - ../metadata-service/restli-servlet-impl/src/main/resources/index/:/index
 
-  kafka-setup:
-    image: linkedin/datahub-kafka-setup:debug
-    build:
-      context: ../
-      dockerfile: ./docker/kafka-setup/Dockerfile
-      args:
-        APP_ENV: dev
-    depends_on:
-      - broker
-      - schema-registry
-
   datahub-gms:
     image: linkedin/datahub-gms:debug
     build:

docker/docker-compose.kafka-setup.yml

@@ -0,0 +1,15 @@
+# Service definitions for Kafka Setup container.
+version: '3.8'
+services:
+
+  # This "container" is a workaround to pre-create topics
+  kafka-setup:
+    build:
+      context: kafka-setup
+    image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
+    env_file: kafka-setup/env/docker.env
+    hostname: kafka-setup
+    container_name: kafka-setup
+    depends_on:
+      - broker
+      - schema-registry

docker/docker-compose.m1.yml

@@ -1,11 +1,5 @@
 services:
-  broker:
-    image: kymeric/cp-kafka:latest
   mysql:
     image: mariadb:10.5.8
-  schema-registry:
-    image: eugenetea/schema-registry-arm64:latest
-  zookeeper:
-    image: kymeric/cp-zookeeper:latest
   neo4j:
     image: neo4j/neo4j-arm64-experimental:4.0.6-arm64

docker/docker-compose.override.yml

@@ -7,7 +7,7 @@ services:
     hostname: mysql
     image: mysql:5.7
     env_file: mysql/env/docker.env
-    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
+    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
     ports:
       - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
     volumes:

docker/docker-compose.tools.yml

@@ -3,7 +3,7 @@
 version: '3.8'
 services:
   kafka-rest-proxy:
-    image: confluentinc/cp-kafka-rest:5.4.0
+    image: confluentinc/cp-kafka-rest:7.2.2
     env_file: kafka-rest-proxy/env/docker.env
     hostname: kafka-rest-proxy
     container_name: kafka-rest-proxy

docker/docker-compose.yml

@@ -7,7 +7,7 @@
 version: '3.8'
 services:
   zookeeper:
-    image: confluentinc/cp-zookeeper:5.4.0
+    image: confluentinc/cp-zookeeper:7.2.2
     env_file: zookeeper/env/docker.env
     hostname: zookeeper
     container_name: zookeeper
@@ -17,7 +17,7 @@ services:
       - zkdata:/var/lib/zookeeper
 
   broker:
-    image: confluentinc/cp-kafka:5.4.0
+    image: confluentinc/cp-kafka:7.2.2
     env_file: broker/env/docker.env
     hostname: broker
     container_name: broker
@@ -28,26 +28,12 @@ services:
     volumes:
       - broker:/var/lib/kafka/data/
 
-  # This "container" is a workaround to pre-create topics
-  kafka-setup:
-    build:
-      dockerfile: ./docker/kafka-setup/Dockerfile
-      context: ../
-    image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
-    env_file: kafka-setup/env/docker.env
-    hostname: kafka-setup
-    container_name: kafka-setup
-    depends_on:
-      - broker
-      - schema-registry
-
   schema-registry:
-    image: confluentinc/cp-schema-registry:5.4.0
+    image: confluentinc/cp-schema-registry:7.2.2
     env_file: schema-registry/env/docker.env
     hostname: schema-registry
     container_name: schema-registry
     depends_on:
-      - zookeeper
       - broker
     ports:
       - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081