fix(standalone mae consumer): support standalone mae consumer without neo4j

docker/datahub-gms/start.sh

@@ -26,10 +26,17 @@ else
     ELASTICSEARCH_PROTOCOL=http
 fi
 
+WAIT_FOR_NEO4J=""
+
+if [[ $GRAPH_SERVICE_IMPL != elasticsearch ]]; then
+  WAIT_FOR_NEO4J=" -wait $NEO4J_HOST "
+fi
+
 dockerize \
   -wait tcp://$EBEAN_DATASOURCE_HOST \
   -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \
   -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT -wait-http-header "$ELASTICSEARCH_AUTH_HEADER" \
+  $WAIT_FOR_NEO4J \
   -timeout 240s \
   java $JAVA_OPTS $JMX_OPTS \
   -jar /jetty-runner.jar \

docker/datahub-mae-consumer/env/docker-without-neo4j.env

@@ -0,0 +1,27 @@
+MAE_CONSUMER_ENABLED=true
+KAFKA_BOOTSTRAP_SERVER=broker:29092
+KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+ELASTICSEARCH_HOST=elasticsearch
+ELASTICSEARCH_PORT=9200
+GMS_HOST=datahub-gms
+GMS_PORT=8080
+GRAPH_SERVICE_IMPL=elasticsearch
+
+# Uncomment to disable persistence of client-side analytics events
+# DATAHUB_ANALYTICS_ENABLED=false
+
+# Uncomment to configure topic names
+# Make sure these names are consistent across the whole deployment
+# KAFKA_TOPIC_NAME=MetadataAuditEvent_v4
+# DATAHUB_USAGE_EVENT_NAME=DataHubUsageEvent_v1
+
+# Uncomment and set these to support SSL connection to Elasticsearch
+# ELASTICSEARCH_USE_SSL=
+# ELASTICSEARCH_SSL_PROTOCOL=
+# ELASTICSEARCH_SSL_SECURE_RANDOM_IMPL=
+# ELASTICSEARCH_SSL_TRUSTSTORE_FILE=
+# ELASTICSEARCH_SSL_TRUSTSTORE_TYPE=
+# ELASTICSEARCH_SSL_TRUSTSTORE_PASSWORD=
+# ELASTICSEARCH_SSL_KEYSTORE_FILE=
+# ELASTICSEARCH_SSL_KEYSTORE_TYPE=
+# ELASTICSEARCH_SSL_KEYSTORE_PASSWORD=

docker/datahub-mae-consumer/env/docker.env

@@ -9,6 +9,7 @@ NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=datahub
 GMS_HOST=datahub-gms
 GMS_PORT=8080
+GRAPH_SERVICE_IMPL=neo4j
 
 # Uncomment to disable persistence of client-side analytics events
 # DATAHUB_ANALYTICS_ENABLED=false
@@ -27,4 +28,4 @@ GMS_PORT=8080
 # ELASTICSEARCH_SSL_TRUSTSTORE_PASSWORD=
 # ELASTICSEARCH_SSL_KEYSTORE_FILE=
 # ELASTICSEARCH_SSL_KEYSTORE_TYPE=
-# ELASTICSEARCH_SSL_KEYSTORE_PASSWORD=
+# ELASTICSEARCH_SSL_KEYSTORE_PASSWORD=

docker/datahub-mae-consumer/start.sh

@@ -26,9 +26,15 @@ else
     ELASTICSEARCH_PROTOCOL=http
 fi
 
-dockerize \
-  -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \
-  -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT -wait-http-header "$ELASTICSEARCH_AUTH_HEADER" \
-  -wait $NEO4J_HOST \
-  -timeout 240s \
-  java $JAVA_OPTS $JMX_OPTS -jar /datahub/datahub-mae-consumer/bin/mae-consumer-job.jar
+WAIT_FOR_NEO4J=""
+
+if [[ $GRAPH_SERVICE_IMPL != elasticsearch ]]; then
+  WAIT_FOR_NEO4J=" -wait $NEO4J_HOST "
+fi
+
+    dockerize \
+      -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \
+      -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT -wait-http-header "$ELASTICSEARCH_AUTH_HEADER" \
+      $WAIT_FOR_NEO4J \
+      -timeout 240s \
+      java $JAVA_OPTS $JMX_OPTS -jar /datahub/datahub-mae-consumer/bin/mae-consumer-job.jar

docker/dev-without-neo4j.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Launches dev instances of DataHub images. See documentation for more details.
+# YOU MUST BUILD VIA GRADLE BEFORE RUNNING THIS.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd $DIR && \
+  COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose \
+    -f docker-compose-without-neo4j.yml \
+    -f docker-compose-without-neo4j.override.yml \
+    -f docker-compose.dev.yml \
+    pull \
+&& \
+  COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -p datahub \
+    -f docker-compose-without-neo4j.yml \
+    -f docker-compose-without-neo4j.override.yml \
+    -f docker-compose.dev.yml \
+    up --build $@

docker/docker-compose.consumers-without-neo4j.yml

@@ -0,0 +1,30 @@
+# Service definitions for standalone Kafka consumer containers.
+version: '3.8'
+services:
+  datahub-mae-consumer:
+    build:
+      context: ../
+      dockerfile: docker/datahub-mae-consumer/Dockerfile
+    image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-head}
+    env_file: datahub-mae-consumer/env/docker-without-neo4j.env
+    hostname: datahub-mae-consumer
+    container_name: datahub-mae-consumer
+    ports:
+      - "9091:9091"
+    depends_on:
+      - kafka-setup
+      - elasticsearch-setup
+
+  datahub-mce-consumer:
+    build:
+      context: ../
+      dockerfile: docker/datahub-mce-consumer/Dockerfile
+    image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-head}
+    env_file: datahub-mce-consumer/env/docker.env
+    hostname: datahub-mce-consumer
+    container_name: datahub-mce-consumer
+    ports:
+      - "9090:9090"
+    depends_on:
+      - kafka-setup
+      - datahub-gms