diff --git a/docker/datahub-gms/start.sh b/docker/datahub-gms/start.sh index 0e4d18f6f71cb3..661cce9dbebacc 100755 --- a/docker/datahub-gms/start.sh +++ b/docker/datahub-gms/start.sh @@ -26,10 +26,17 @@ else ELASTICSEARCH_PROTOCOL=http fi +WAIT_FOR_NEO4J="" + +if [[ $GRAPH_SERVICE_IMPL != elasticsearch ]]; then + WAIT_FOR_NEO4J=" -wait $NEO4J_HOST " +fi + dockerize \ -wait tcp://$EBEAN_DATASOURCE_HOST \ -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \ -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT -wait-http-header "$ELASTICSEARCH_AUTH_HEADER" \ + $WAIT_FOR_NEO4J \ -timeout 240s \ java $JAVA_OPTS $JMX_OPTS \ -jar /jetty-runner.jar \ diff --git a/docker/datahub-mae-consumer/env/docker-without-neo4j.env b/docker/datahub-mae-consumer/env/docker-without-neo4j.env new file mode 100644 index 00000000000000..c8c44f33c3bc90 --- /dev/null +++ b/docker/datahub-mae-consumer/env/docker-without-neo4j.env @@ -0,0 +1,27 @@ +MAE_CONSUMER_ENABLED=true +KAFKA_BOOTSTRAP_SERVER=broker:29092 +KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 +ELASTICSEARCH_HOST=elasticsearch +ELASTICSEARCH_PORT=9200 +GMS_HOST=datahub-gms +GMS_PORT=8080 +GRAPH_SERVICE_IMPL=elasticsearch + +# Uncomment to disable persistence of client-side analytics events +# DATAHUB_ANALYTICS_ENABLED=false + +# Uncomment to configure topic names +# Make sure these names are consistent across the whole deployment +# KAFKA_TOPIC_NAME=MetadataAuditEvent_v4 +# DATAHUB_USAGE_EVENT_NAME=DataHubUsageEvent_v1 + +# Uncomment and set these to support SSL connection to Elasticsearch +# ELASTICSEARCH_USE_SSL= +# ELASTICSEARCH_SSL_PROTOCOL= +# ELASTICSEARCH_SSL_SECURE_RANDOM_IMPL= +# ELASTICSEARCH_SSL_TRUSTSTORE_FILE= +# ELASTICSEARCH_SSL_TRUSTSTORE_TYPE= +# ELASTICSEARCH_SSL_TRUSTSTORE_PASSWORD= +# ELASTICSEARCH_SSL_KEYSTORE_FILE= +# ELASTICSEARCH_SSL_KEYSTORE_TYPE= +# ELASTICSEARCH_SSL_KEYSTORE_PASSWORD= diff --git a/docker/datahub-mae-consumer/env/docker.env b/docker/datahub-mae-consumer/env/docker.env index e7f192901ea0e3..873b7d3d8f0601 100644 --- a/docker/datahub-mae-consumer/env/docker.env +++ b/docker/datahub-mae-consumer/env/docker.env @@ -9,6 +9,7 @@ NEO4J_USERNAME=neo4j NEO4J_PASSWORD=datahub GMS_HOST=datahub-gms GMS_PORT=8080 +GRAPH_SERVICE_IMPL=neo4j # Uncomment to disable persistence of client-side analytics events # DATAHUB_ANALYTICS_ENABLED=false @@ -27,4 +28,4 @@ GMS_PORT=8080 # ELASTICSEARCH_SSL_TRUSTSTORE_PASSWORD= # ELASTICSEARCH_SSL_KEYSTORE_FILE= # ELASTICSEARCH_SSL_KEYSTORE_TYPE= -# ELASTICSEARCH_SSL_KEYSTORE_PASSWORD= \ No newline at end of file +# ELASTICSEARCH_SSL_KEYSTORE_PASSWORD= diff --git a/docker/datahub-mae-consumer/start.sh b/docker/datahub-mae-consumer/start.sh index e0e232f25d68b4..e0d4d19fcf4f15 100755 --- a/docker/datahub-mae-consumer/start.sh +++ b/docker/datahub-mae-consumer/start.sh @@ -26,9 +26,15 @@ else ELASTICSEARCH_PROTOCOL=http fi -dockerize \ - -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \ - -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT -wait-http-header "$ELASTICSEARCH_AUTH_HEADER" \ - -wait $NEO4J_HOST \ - -timeout 240s \ - java $JAVA_OPTS $JMX_OPTS -jar /datahub/datahub-mae-consumer/bin/mae-consumer-job.jar \ No newline at end of file +WAIT_FOR_NEO4J="" + +if [[ $GRAPH_SERVICE_IMPL != elasticsearch ]]; then + WAIT_FOR_NEO4J=" -wait $NEO4J_HOST " +fi + + dockerize \ + -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \ + -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT -wait-http-header "$ELASTICSEARCH_AUTH_HEADER" \ + $WAIT_FOR_NEO4J \ + -timeout 240s \ + java $JAVA_OPTS $JMX_OPTS -jar /datahub/datahub-mae-consumer/bin/mae-consumer-job.jar diff --git a/docker/dev-without-neo4j.sh b/docker/dev-without-neo4j.sh new file mode 100755 index 00000000000000..76b8c94987e653 --- /dev/null +++ b/docker/dev-without-neo4j.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Launches dev instances of DataHub images. See documentation for more details. +# YOU MUST BUILD VIA GRADLE BEFORE RUNNING THIS. +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $DIR && \ + COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose \ + -f docker-compose-without-neo4j.yml \ + -f docker-compose-without-neo4j.override.yml \ + -f docker-compose.dev.yml \ + pull \ +&& \ + COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -p datahub \ + -f docker-compose-without-neo4j.yml \ + -f docker-compose-without-neo4j.override.yml \ + -f docker-compose.dev.yml \ + up --build $@ diff --git a/docker/docker-compose.consumers-without-neo4j.yml b/docker/docker-compose.consumers-without-neo4j.yml new file mode 100644 index 00000000000000..956b3adcc9fe48 --- /dev/null +++ b/docker/docker-compose.consumers-without-neo4j.yml @@ -0,0 +1,30 @@ +# Service definitions for standalone Kafka consumer containers. +version: '3.8' +services: + datahub-mae-consumer: + build: + context: ../ + dockerfile: docker/datahub-mae-consumer/Dockerfile + image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-head} + env_file: datahub-mae-consumer/env/docker-without-neo4j.env + hostname: datahub-mae-consumer + container_name: datahub-mae-consumer + ports: + - "9091:9091" + depends_on: + - kafka-setup + - elasticsearch-setup + + datahub-mce-consumer: + build: + context: ../ + dockerfile: docker/datahub-mce-consumer/Dockerfile + image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-head} + env_file: datahub-mce-consumer/env/docker.env + hostname: datahub-mce-consumer + container_name: datahub-mce-consumer + ports: + - "9090:9090" + depends_on: + - kafka-setup + - datahub-gms