From 86879eecf7724e2698431bfff4e51bb51c4b32ee Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Wed, 18 Jan 2023 19:33:04 +0000 Subject: [PATCH 01/22] feat(quickstart): Remove kafka-setup as a hard deployment requirement --- docker/docker-compose-with-cassandra.yml | 12 ------- docker/docker-compose-without-neo4j.yml | 12 ------- docker/docker-compose.dev.yml | 11 ------ docker/docker-compose.kafka-setup.yml | 15 ++++++++ ...ocker-compose-without-neo4j.quickstart.yml | 10 ------ ...ose.consumers-without-neo4j.quickstart.yml | 10 ++---- .../docker-compose.consumers.quickstart.yml | 20 +++-------- .../docker-compose.kafka-setup.quickstart.yml | 12 +++++++ docker/quickstart/generate_and_compare.sh | 1 + .../quickstart/generate_docker_quickstart.sh | 1 + .../src/datahub/cli/docker_check.py | 1 - .../src/datahub/cli/docker_cli.py | 36 ++++++++++++++++++- 12 files changed, 71 insertions(+), 70 deletions(-) create mode 100644 docker/docker-compose.kafka-setup.yml create mode 100644 docker/quickstart/docker-compose.kafka-setup.quickstart.yml diff --git a/docker/docker-compose-with-cassandra.yml b/docker/docker-compose-with-cassandra.yml index e7d2ae9eb842b..c9d316c079c69 100644 --- a/docker/docker-compose-with-cassandra.yml +++ b/docker/docker-compose-with-cassandra.yml @@ -29,18 +29,6 @@ services: volumes: - broker:/var/lib/kafka/data/ - # This "container" is a workaround to pre-create topics - kafka-setup: - build: - context: kafka-setup - image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} - env_file: kafka-setup/env/docker.env - hostname: kafka-setup - container_name: kafka-setup - depends_on: - - broker - - schema-registry - schema-registry: image: confluentinc/cp-schema-registry:5.4.0 env_file: schema-registry/env/docker.env diff --git a/docker/docker-compose-without-neo4j.yml b/docker/docker-compose-without-neo4j.yml index d8eb39f49e39d..6e617ffeeef80 100644 --- a/docker/docker-compose-without-neo4j.yml +++ b/docker/docker-compose-without-neo4j.yml @@ -26,18 +26,6 @@ services: ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 - # This "container" is a workaround to pre-create topics - kafka-setup: - build: - context: kafka-setup - image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} - env_file: kafka-setup/env/docker.env - hostname: kafka-setup - container_name: kafka-setup - depends_on: - - broker - - schema-registry - schema-registry: image: confluentinc/cp-schema-registry:5.4.0 env_file: schema-registry/env/docker.env diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 3dbd9ba34bda4..581a8a49bda4c 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -24,17 +24,6 @@ services: - ./elasticsearch-setup/create-indices.sh:/create-indices.sh - ../metadata-service/restli-servlet-impl/src/main/resources/index/:/index - kafka-setup: - image: linkedin/datahub-kafka-setup:debug - build: - context: ../ - dockerfile: ./docker/kafka-setup/Dockerfile - args: - APP_ENV: dev - depends_on: - - broker - - schema-registry - datahub-gms: image: linkedin/datahub-gms:debug build: diff --git a/docker/docker-compose.kafka-setup.yml b/docker/docker-compose.kafka-setup.yml new file mode 100644 index 0000000000000..f4e5a743ce0b5 --- /dev/null +++ b/docker/docker-compose.kafka-setup.yml @@ -0,0 +1,15 @@ +# Service definitions for Kafka Setup container. +version: '3.8' +services: + + # This "container" is a workaround to pre-create topics + kafka-setup: + build: + context: kafka-setup + image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} + env_file: kafka-setup/env/docker.env + hostname: kafka-setup + container_name: kafka-setup + depends_on: + - broker + - schema-registry \ No newline at end of file diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index e815ff60ca42b..acb36d4703e4e 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -133,16 +133,6 @@ services: - ELASTICSEARCH_PROTOCOL=http hostname: elasticsearch-setup image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} - kafka-setup: - container_name: kafka-setup - depends_on: - - broker - - schema-registry - environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - hostname: kafka-setup - image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} mysql: command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin container_name: mysql diff --git a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml index eb645a6abae2f..63a6638890d12 100644 --- a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml @@ -25,15 +25,13 @@ services: container_name: datahub-mce-consumer environment: - MCE_CONSUMER_ENABLED=true - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - EBEAN_DATASOURCE_USERNAME=datahub - EBEAN_DATASOURCE_PASSWORD=datahub - EBEAN_DATASOURCE_HOST=mysql:3306 - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 - ES_BULK_REFRESH_POLICY=WAIT_UNTIL @@ -43,10 +41,6 @@ services: - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - ENTITY_SERVICE_ENABLE_RETENTION=true - - MAE_CONSUMER_ENABLED=false - - PE_CONSUMER_ENABLED=false - - UI_INGESTION_ENABLED=false hostname: datahub-mce-consumer image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head} ports: diff --git a/docker/quickstart/docker-compose.consumers.quickstart.yml b/docker/quickstart/docker-compose.consumers.quickstart.yml index 7edfd52026473..1ae587fc04cfa 100644 --- a/docker/quickstart/docker-compose.consumers.quickstart.yml +++ b/docker/quickstart/docker-compose.consumers.quickstart.yml @@ -31,32 +31,22 @@ services: container_name: datahub-mce-consumer environment: - MCE_CONSUMER_ENABLED=true - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - EBEAN_DATASOURCE_USERNAME=datahub - EBEAN_DATASOURCE_PASSWORD=datahub - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - NEO4J_HOST=http://neo4j:7474 - - NEO4J_URI=bolt://neo4j - - NEO4J_USERNAME=neo4j - - NEO4J_PASSWORD=datahub - - JAVA_OPTS=-Xms1g -Xmx1g - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - GRAPH_SERVICE_IMPL=neo4j + - GRAPH_SERVICE_IMPL=elasticsearch + - JAVA_OPTS=-Xms1g -Xmx1g - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - ENTITY_SERVICE_ENABLE_RETENTION=true - - MAE_CONSUMER_ENABLED=false - - PE_CONSUMER_ENABLED=false - - UI_INGESTION_ENABLED=false hostname: datahub-mce-consumer image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head} ports: diff --git a/docker/quickstart/docker-compose.kafka-setup.quickstart.yml b/docker/quickstart/docker-compose.kafka-setup.quickstart.yml new file mode 100644 index 0000000000000..2818806e8d0b5 --- /dev/null +++ b/docker/quickstart/docker-compose.kafka-setup.quickstart.yml @@ -0,0 +1,12 @@ +services: + kafka-setup: + container_name: kafka-setup + depends_on: + - broker + - schema-registry + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + hostname: kafka-setup + image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} +version: '2.3' diff --git a/docker/quickstart/generate_and_compare.sh b/docker/quickstart/generate_and_compare.sh index 4e7c26b2655a2..66251acce25bb 100755 --- a/docker/quickstart/generate_and_compare.sh +++ b/docker/quickstart/generate_and_compare.sh @@ -17,6 +17,7 @@ python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../dock python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml temp.monitoring.quickstart.yml python generate_docker_quickstart.py ../docker-compose.consumers.yml temp.consumers.quickstart.yml python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml temp.consumers-without-neo4j.quickstart.yml +python generate_docker_quickstart.py ../docker-compose.kafka-setup.yml temp.kafka-setup.yml for flavour in "${FLAVOURS[@]}" do diff --git a/docker/quickstart/generate_docker_quickstart.sh b/docker/quickstart/generate_docker_quickstart.sh index aa3c767430df1..3a3846349843d 100755 --- a/docker/quickstart/generate_docker_quickstart.sh +++ b/docker/quickstart/generate_docker_quickstart.sh @@ -14,3 +14,4 @@ python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../dock python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml docker-compose.monitoring.quickstart.yml python generate_docker_quickstart.py ../docker-compose.consumers.yml docker-compose.consumers.quickstart.yml python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml docker-compose.consumers-without-neo4j.quickstart.yml +python generate_docker_quickstart.py ../docker-compose.kafka-setup.yml docker-compose.kafka-setup.quickstart.yml diff --git a/metadata-ingestion/src/datahub/cli/docker_check.py b/metadata-ingestion/src/datahub/cli/docker_check.py index 5c7a86357118e..81eae41869d68 100644 --- a/metadata-ingestion/src/datahub/cli/docker_check.py +++ b/metadata-ingestion/src/datahub/cli/docker_check.py @@ -9,7 +9,6 @@ "elasticsearch", "datahub-gms", "datahub-frontend-react", - "kafka-setup", "schema-registry", "broker", "zookeeper", diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py index 7bbd7577b5880..8dd757b33bda9 100644 --- a/metadata-ingestion/src/datahub/cli/docker_cli.py +++ b/metadata-ingestion/src/datahub/cli/docker_cli.py @@ -52,7 +52,9 @@ ELASTIC_CONSUMERS_QUICKSTART_COMPOSE_FILE = ( "docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml" ) - +KAFKA_SETUP_QUICKSTART_COMPOSE_FILE = ( + "docker/quickstart/docker-compose.kafka-setup.quickstart.yml" +) NEO4J_AND_ELASTIC_QUICKSTART_COMPOSE_URL = ( f"{DOCKER_COMPOSE_BASE}/{NEO4J_AND_ELASTIC_QUICKSTART_COMPOSE_FILE}" ) @@ -581,6 +583,13 @@ def detect_quickstart_arch(arch: Optional[str]) -> Architectures: default=False, help="Launches MAE & MCE consumers as stand alone docker containers", ) +@click.option( + "--kafka-setup", + required=False, + is_flag=True, + default=False, + help="Launches Kafka setup job as part of the compose deployment", +) @click.option( "--arch", required=False, @@ -608,6 +617,7 @@ def quickstart( restore_indices: bool, no_restore_indices: bool, standalone_consumers: bool, + kafka_setup: bool, arch: Optional[str], ) -> None: """Start an instance of DataHub locally using docker-compose. @@ -714,6 +724,30 @@ def quickstart( tmp_file.write(quickstart_download_response.content) logger.debug(f"Copied to {path}") + if kafka_setup: + kafka_setup_github_file = ( + f"{DOCKER_COMPOSE_BASE}/{KAFKA_SETUP_QUICKSTART_COMPOSE_FILE}" + ) + + default_consumer_compose_file = ( + Path(DATAHUB_ROOT_FOLDER) / "quickstart/docker-compose.consumers.yml" + ) + with open( + default_consumer_compose_file, "wb" + ) if default_consumer_compose_file else tempfile.NamedTemporaryFile( + suffix=".yml", delete=False + ) as tmp_file: + path = pathlib.Path(tmp_file.name) + quickstart_compose_file.append(path) + click.echo( + f"Fetching consumer docker-compose file {kafka_setup_github_file} from GitHub" + ) + # Download the quickstart docker-compose file from GitHub. + quickstart_download_response = request_session.get(kafka_setup_github_file) + quickstart_download_response.raise_for_status() + tmp_file.write(quickstart_download_response.content) + logger.debug(f"Copied to {path}") + # set version _set_environment_variables( version=version, From 7927b816835f76c18d498cfc692d048f22e091f4 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Thu, 19 Jan 2023 11:34:32 +0000 Subject: [PATCH 02/22] Update quickstart --- docker/docker-compose.yml | 13 ------------- .../docker-compose-without-neo4j-m1.quickstart.yml | 10 ---------- docker/quickstart/docker-compose.quickstart.yml | 10 ---------- docker/quickstart/generate_docker_quickstart.sh | 1 + 4 files changed, 1 insertion(+), 33 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 764ebf2e555dd..a1eff1e752ded 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -28,19 +28,6 @@ services: volumes: - broker:/var/lib/kafka/data/ - # This "container" is a workaround to pre-create topics - kafka-setup: - build: - dockerfile: ./docker/kafka-setup/Dockerfile - context: ../ - image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} - env_file: kafka-setup/env/docker.env - hostname: kafka-setup - container_name: kafka-setup - depends_on: - - broker - - schema-registry - schema-registry: image: confluentinc/cp-schema-registry:5.4.0 env_file: schema-registry/env/docker.env diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index c549296d58064..bf29a265410f1 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -131,16 +131,6 @@ services: - ELASTICSEARCH_PROTOCOL=http hostname: elasticsearch-setup image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} - kafka-setup: - container_name: kafka-setup - depends_on: - - broker - - schema-registry - environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - hostname: kafka-setup - image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} mysql: command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password container_name: mysql diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 01e3b799265a1..b387263941e3a 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -141,16 +141,6 @@ services: - ELASTICSEARCH_PROTOCOL=http hostname: elasticsearch-setup image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} - kafka-setup: - container_name: kafka-setup - depends_on: - - broker - - schema-registry - environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - hostname: kafka-setup - image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} mysql: command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin container_name: mysql diff --git a/docker/quickstart/generate_docker_quickstart.sh b/docker/quickstart/generate_docker_quickstart.sh index 3a3846349843d..ac39c07702c56 100755 --- a/docker/quickstart/generate_docker_quickstart.sh +++ b/docker/quickstart/generate_docker_quickstart.sh @@ -15,3 +15,4 @@ python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml python generate_docker_quickstart.py ../docker-compose.consumers.yml docker-compose.consumers.quickstart.yml python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml docker-compose.consumers-without-neo4j.quickstart.yml python generate_docker_quickstart.py ../docker-compose.kafka-setup.yml docker-compose.kafka-setup.quickstart.yml +# TODO docker-compose-without-neo4j-m1.quickstart.yml is not getting generated \ No newline at end of file From 4e66f4240d0716bfeda21861490294789a499dba Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Thu, 19 Jan 2023 11:54:46 +0000 Subject: [PATCH 03/22] Fix mce env vars --- docker/datahub-mce-consumer/env/docker.env | 6 +++++- docker/docker-compose.consumers-without-neo4j.yml | 2 ++ docker/docker-compose.consumers.yml | 2 ++ .../docker-compose.consumers-without-neo4j.quickstart.yml | 6 ++++++ docker/quickstart/docker-compose.consumers.quickstart.yml | 6 ++++++ docker/quickstart/generate_and_compare.sh | 1 - 6 files changed, 21 insertions(+), 2 deletions(-) diff --git a/docker/datahub-mce-consumer/env/docker.env b/docker/datahub-mce-consumer/env/docker.env index f11ce88f72cdd..89181109e4fc0 100644 --- a/docker/datahub-mce-consumer/env/docker.env +++ b/docker/datahub-mce-consumer/env/docker.env @@ -15,7 +15,11 @@ JAVA_OPTS=-Xms1g -Xmx1g ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml DATAHUB_SYSTEM_CLIENT_ID=__datahub_system DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - +DATAHUB_TELEMETRY_ENABLED=true +ENTITY_SERVICE_ENABLE_RETENTION=true +MAE_CONSUMER_ENABLED=false +PE_CONSUMER_ENABLED=false +UI_INGESTION_ENABLED=false # Uncomment to configure kafka topic names # Make sure these names are consistent across the whole deployment # METADATA_CHANGE_PROPOSAL_TOPIC_NAME=MetadataChangeProposal_v1 diff --git a/docker/docker-compose.consumers-without-neo4j.yml b/docker/docker-compose.consumers-without-neo4j.yml index d80beae8bfad3..c0ed3c99ffb05 100644 --- a/docker/docker-compose.consumers-without-neo4j.yml +++ b/docker/docker-compose.consumers-without-neo4j.yml @@ -25,5 +25,7 @@ services: env_file: datahub-mce-consumer/env/docker.env hostname: datahub-mce-consumer container_name: datahub-mce-consumer + environment: + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} ports: - "9090:9090" diff --git a/docker/docker-compose.consumers.yml b/docker/docker-compose.consumers.yml index ffe9827bc9c22..92e3ae1845eef 100644 --- a/docker/docker-compose.consumers.yml +++ b/docker/docker-compose.consumers.yml @@ -27,5 +27,7 @@ services: env_file: datahub-mce-consumer/env/docker.env hostname: datahub-mce-consumer container_name: datahub-mce-consumer + environment: + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} ports: - "9090:9090" diff --git a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml index 63a6638890d12..b5194002f2b68 100644 --- a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml @@ -24,6 +24,7 @@ services: datahub-mce-consumer: container_name: datahub-mce-consumer environment: + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - MCE_CONSUMER_ENABLED=true - EBEAN_DATASOURCE_USERNAME=datahub - EBEAN_DATASOURCE_PASSWORD=datahub @@ -41,6 +42,11 @@ services: - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - DATAHUB_TELEMETRY_ENABLED=true + - ENTITY_SERVICE_ENABLE_RETENTION=true + - MAE_CONSUMER_ENABLED=false + - PE_CONSUMER_ENABLED=false + - UI_INGESTION_ENABLED=false hostname: datahub-mce-consumer image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head} ports: diff --git a/docker/quickstart/docker-compose.consumers.quickstart.yml b/docker/quickstart/docker-compose.consumers.quickstart.yml index 1ae587fc04cfa..8ba128327858e 100644 --- a/docker/quickstart/docker-compose.consumers.quickstart.yml +++ b/docker/quickstart/docker-compose.consumers.quickstart.yml @@ -30,6 +30,7 @@ services: datahub-mce-consumer: container_name: datahub-mce-consumer environment: + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - MCE_CONSUMER_ENABLED=true - EBEAN_DATASOURCE_USERNAME=datahub - EBEAN_DATASOURCE_PASSWORD=datahub @@ -47,6 +48,11 @@ services: - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - DATAHUB_TELEMETRY_ENABLED=true + - ENTITY_SERVICE_ENABLE_RETENTION=true + - MAE_CONSUMER_ENABLED=false + - PE_CONSUMER_ENABLED=false + - UI_INGESTION_ENABLED=false hostname: datahub-mce-consumer image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head} ports: diff --git a/docker/quickstart/generate_and_compare.sh b/docker/quickstart/generate_and_compare.sh index 66251acce25bb..4e7c26b2655a2 100755 --- a/docker/quickstart/generate_and_compare.sh +++ b/docker/quickstart/generate_and_compare.sh @@ -17,7 +17,6 @@ python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../dock python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml temp.monitoring.quickstart.yml python generate_docker_quickstart.py ../docker-compose.consumers.yml temp.consumers.quickstart.yml python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml temp.consumers-without-neo4j.quickstart.yml -python generate_docker_quickstart.py ../docker-compose.kafka-setup.yml temp.kafka-setup.yml for flavour in "${FLAVOURS[@]}" do From e8fd760e7083d3b068e14bc355fd2a400d27969a Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Thu, 19 Jan 2023 11:55:21 +0000 Subject: [PATCH 04/22] more fixes --- docker/datahub-mce-consumer/env/docker.env | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/datahub-mce-consumer/env/docker.env b/docker/datahub-mce-consumer/env/docker.env index 89181109e4fc0..413a37e82f37b 100644 --- a/docker/datahub-mce-consumer/env/docker.env +++ b/docker/datahub-mce-consumer/env/docker.env @@ -20,6 +20,7 @@ ENTITY_SERVICE_ENABLE_RETENTION=true MAE_CONSUMER_ENABLED=false PE_CONSUMER_ENABLED=false UI_INGESTION_ENABLED=false + # Uncomment to configure kafka topic names # Make sure these names are consistent across the whole deployment # METADATA_CHANGE_PROPOSAL_TOPIC_NAME=MetadataChangeProposal_v1 From b8d3c777c686e04fd811f542134b918d06e31427 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Thu, 19 Jan 2023 12:03:03 +0000 Subject: [PATCH 05/22] Fix list merging --- docker/datahub-mce-consumer/env/docker.env | 1 - ...docker-compose.consumers-without-neo4j.yml | 1 + docker/docker-compose.consumers.yml | 1 + ...ocker-compose-without-neo4j.quickstart.yml | 32 +++++++++---------- ...ose.consumers-without-neo4j.quickstart.yml | 2 +- .../docker-compose.consumers.quickstart.yml | 2 +- .../quickstart/docker-compose.quickstart.yml | 1 + .../quickstart/generate_docker_quickstart.py | 9 ++++-- .../quickstart/generate_docker_quickstart.sh | 4 +-- 9 files changed, 29 insertions(+), 24 deletions(-) diff --git a/docker/datahub-mce-consumer/env/docker.env b/docker/datahub-mce-consumer/env/docker.env index 413a37e82f37b..faaa2b7cd4cc3 100644 --- a/docker/datahub-mce-consumer/env/docker.env +++ b/docker/datahub-mce-consumer/env/docker.env @@ -15,7 +15,6 @@ JAVA_OPTS=-Xms1g -Xmx1g ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml DATAHUB_SYSTEM_CLIENT_ID=__datahub_system DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing -DATAHUB_TELEMETRY_ENABLED=true ENTITY_SERVICE_ENABLE_RETENTION=true MAE_CONSUMER_ENABLED=false PE_CONSUMER_ENABLED=false diff --git a/docker/docker-compose.consumers-without-neo4j.yml b/docker/docker-compose.consumers-without-neo4j.yml index c0ed3c99ffb05..cc31c722dbc99 100644 --- a/docker/docker-compose.consumers-without-neo4j.yml +++ b/docker/docker-compose.consumers-without-neo4j.yml @@ -27,5 +27,6 @@ services: container_name: datahub-mce-consumer environment: - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} ports: - "9090:9090" diff --git a/docker/docker-compose.consumers.yml b/docker/docker-compose.consumers.yml index 92e3ae1845eef..2dc7c78366357 100644 --- a/docker/docker-compose.consumers.yml +++ b/docker/docker-compose.consumers.yml @@ -29,5 +29,6 @@ services: container_name: datahub-mce-consumer environment: - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} ports: - "9090:9090" diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index acb36d4703e4e..bffb0721707ed 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -74,29 +74,29 @@ services: depends_on: - mysql environment: - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - MAE_CONSUMER_ENABLED=true + - ENTITY_SERVICE_ENABLE_RETENTION=true - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - EBEAN_DATASOURCE_USERNAME=datahub + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - JAVA_OPTS=-Xms1g -Xmx1g + - MCE_CONSUMER_ENABLED=true + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - UI_INGESTION_ENABLED=true - EBEAN_DATASOURCE_PASSWORD=datahub - - EBEAN_DATASOURCE_HOST=mysql:3306 + - ELASTICSEARCH_HOST=elasticsearch - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - GRAPH_SERVICE_IMPL=elasticsearch + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_USERNAME=datahub - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - GRAPH_SERVICE_IMPL=elasticsearch - - JAVA_OPTS=-Xms1g -Xmx1g - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - MAE_CONSUMER_ENABLED=true - - MCE_CONSUMER_ENABLED=true - PE_CONSUMER_ENABLED=true - - UI_INGESTION_ENABLED=true - - ENTITY_SERVICE_ENABLE_RETENTION=true + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: diff --git a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml index b5194002f2b68..ae73957ea5a36 100644 --- a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml @@ -25,6 +25,7 @@ services: container_name: datahub-mce-consumer environment: - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - MCE_CONSUMER_ENABLED=true - EBEAN_DATASOURCE_USERNAME=datahub - EBEAN_DATASOURCE_PASSWORD=datahub @@ -42,7 +43,6 @@ services: - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - DATAHUB_TELEMETRY_ENABLED=true - ENTITY_SERVICE_ENABLE_RETENTION=true - MAE_CONSUMER_ENABLED=false - PE_CONSUMER_ENABLED=false diff --git a/docker/quickstart/docker-compose.consumers.quickstart.yml b/docker/quickstart/docker-compose.consumers.quickstart.yml index 8ba128327858e..49c16932cc844 100644 --- a/docker/quickstart/docker-compose.consumers.quickstart.yml +++ b/docker/quickstart/docker-compose.consumers.quickstart.yml @@ -31,6 +31,7 @@ services: container_name: datahub-mce-consumer environment: - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - MCE_CONSUMER_ENABLED=true - EBEAN_DATASOURCE_USERNAME=datahub - EBEAN_DATASOURCE_PASSWORD=datahub @@ -48,7 +49,6 @@ services: - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - DATAHUB_TELEMETRY_ENABLED=true - ENTITY_SERVICE_ENABLE_RETENTION=true - MAE_CONSUMER_ENABLED=false - PE_CONSUMER_ENABLED=false diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index b387263941e3a..e203b725c6687 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -75,6 +75,7 @@ services: container_name: datahub-gms depends_on: - mysql + - neo4j environment: - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} diff --git a/docker/quickstart/generate_docker_quickstart.py b/docker/quickstart/generate_docker_quickstart.py index 4888adda2d038..7d7ffacc8cfb6 100644 --- a/docker/quickstart/generate_docker_quickstart.py +++ b/docker/quickstart/generate_docker_quickstart.py @@ -1,5 +1,4 @@ import os -from collections import OrderedDict from collections.abc import Mapping import click @@ -27,10 +26,14 @@ def dict_merge(dct, merge_dct): for k, v in merge_dct.items(): if k in dct and isinstance(dct[k], dict) and isinstance(merge_dct[k], Mapping): dict_merge(dct[k], merge_dct[k]) + elif k in dct and isinstance(dct[k], list): + a = set(dct[k]) + b = set(merge_dct[k]) + if a != b: + dct[k] = list(a.union(b)) else: dct[k] = merge_dct[k] - def modify_docker_config(base_path, docker_yaml_config): # 0. Filter out services to be omitted. for key in list(docker_yaml_config["services"]): @@ -80,7 +83,7 @@ def modify_docker_config(base_path, docker_yaml_config): elif volumes[i].startswith("./"): volumes[i] = "." + volumes[i] - # 9. Set docker compose version to 2. + # 10. Set docker compose version to 2. # We need at least this version, since we use features like start_period for # healthchecks and shell-like variable interpolation. docker_yaml_config["version"] = "2.3" diff --git a/docker/quickstart/generate_docker_quickstart.sh b/docker/quickstart/generate_docker_quickstart.sh index ac39c07702c56..49d2f28751622 100755 --- a/docker/quickstart/generate_docker_quickstart.sh +++ b/docker/quickstart/generate_docker_quickstart.sh @@ -12,7 +12,7 @@ pip install -r requirements.txt python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml docker-compose.quickstart.yml python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml docker-compose-without-neo4j.quickstart.yml python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml docker-compose.monitoring.quickstart.yml -python generate_docker_quickstart.py ../docker-compose.consumers.yml docker-compose.consumers.quickstart.yml -python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml docker-compose.consumers-without-neo4j.quickstart.yml +python generate_docker_quickstart.py ../docker-compose.consumers.yml docker-compose.consumers.quickstart.yml # Investigate +python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml docker-compose.consumers-without-neo4j.quickstart.yml # investigate python generate_docker_quickstart.py ../docker-compose.kafka-setup.yml docker-compose.kafka-setup.quickstart.yml # TODO docker-compose-without-neo4j-m1.quickstart.yml is not getting generated \ No newline at end of file From 959de3860f0099c52bf9a929c537890820d7f8d9 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Thu, 19 Jan 2023 14:29:45 +0000 Subject: [PATCH 06/22] Dedup env vars & make neo4j consumers consistent --- docker/datahub-mce-consumer/env/docker.env | 2 +- docker/docker-compose.consumers.yml | 9 +++- ...ocker-compose-without-neo4j.quickstart.yml | 43 ++++++++----------- ...ose.consumers-without-neo4j.quickstart.yml | 2 +- .../docker-compose.consumers.quickstart.yml | 16 ++++--- .../quickstart/docker-compose.quickstart.yml | 9 ---- .../quickstart/generate_docker_quickstart.py | 21 +++++++++ 7 files changed, 57 insertions(+), 45 deletions(-) diff --git a/docker/datahub-mce-consumer/env/docker.env b/docker/datahub-mce-consumer/env/docker.env index faaa2b7cd4cc3..73b981d988307 100644 --- a/docker/datahub-mce-consumer/env/docker.env +++ b/docker/datahub-mce-consumer/env/docker.env @@ -2,7 +2,7 @@ MCE_CONSUMER_ENABLED=true EBEAN_DATASOURCE_USERNAME=datahub EBEAN_DATASOURCE_PASSWORD=datahub EBEAN_DATASOURCE_HOST=mysql:3306 -EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 +EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver KAFKA_BOOTSTRAP_SERVER=broker:29092 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 diff --git a/docker/docker-compose.consumers.yml b/docker/docker-compose.consumers.yml index 2dc7c78366357..f7210be9c8f20 100644 --- a/docker/docker-compose.consumers.yml +++ b/docker/docker-compose.consumers.yml @@ -11,7 +11,7 @@ services: context: ../ dockerfile: docker/datahub-mae-consumer/Dockerfile image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head} - env_file: datahub-mae-consumer/env/docker.env + env_file: datahub-mae-consumer/env/docker-without-neo4j.env hostname: datahub-mae-consumer container_name: datahub-mae-consumer ports: @@ -30,5 +30,12 @@ services: environment: - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - NEO4J_HOST=http://neo4j:7474 + - NEO4J_URI=bolt://neo4j + - NEO4J_USERNAME=neo4j + - NEO4J_PASSWORD=datahub + - GRAPH_SERVICE_IMPL=neo4j ports: - "9090:9090" + depends_on: + - neo4j diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index bffb0721707ed..4f3be96a66955 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -34,15 +34,6 @@ services: - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT - DATAHUB_ACTIONS_SLACK_ENABLED - - DATAHUB_ACTIONS_SLACK_DATAHUB_BASE_URL - - DATAHUB_ACTIONS_SLACK_BOT_TOKEN - - DATAHUB_ACTIONS_SLACK_SIGNING_SECRET - - DATAHUB_ACTIONS_SLACK_CHANNEL - - DATAHUB_ACTIONS_SLACK_SUPPRESS_SYSTEM_ACTIVITY - - DATAHUB_ACTIONS_TEAMS_ENABLED - - DATAHUB_ACTIONS_TEAMS_DATAHUB_BASE_URL - - DATAHUB_ACTIONS_TEAMS_WEBHOOK_URL - - DATAHUB_ACTIONS_TEAMS_SUPPRESS_SYSTEM_ACTIVITY hostname: actions image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} restart: on-failure:5 @@ -74,28 +65,28 @@ services: depends_on: - mysql environment: - - MAE_CONSUMER_ENABLED=true - - ENTITY_SERVICE_ENABLE_RETENTION=true - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - JAVA_OPTS=-Xms1g -Xmx1g - - MCE_CONSUMER_ENABLED=true - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - UI_INGESTION_ENABLED=true - - EBEAN_DATASOURCE_PASSWORD=datahub - - ELASTICSEARCH_HOST=elasticsearch - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - GRAPH_SERVICE_IMPL=elasticsearch - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_PORT=9200 - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - EBEAN_DATASOURCE_HOST=mysql:3306 + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - PE_CONSUMER_ENABLED=true - EBEAN_DATASOURCE_USERNAME=datahub + - EBEAN_DATASOURCE_PASSWORD=datahub + - UI_INGESTION_ENABLED=true + - ELASTICSEARCH_HOST=elasticsearch + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - JAVA_OPTS=-Xms1g -Xmx1g + - ENTITY_SERVICE_ENABLE_RETENTION=true + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - GRAPH_SERVICE_IMPL=elasticsearch - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - PE_CONSUMER_ENABLED=true + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - MAE_CONSUMER_ENABLED=true - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - ELASTICSEARCH_PORT=9200 + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - MCE_CONSUMER_ENABLED=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} diff --git a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml index ae73957ea5a36..cee66989456e7 100644 --- a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml @@ -30,7 +30,7 @@ services: - EBEAN_DATASOURCE_USERNAME=datahub - EBEAN_DATASOURCE_PASSWORD=datahub - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - KAFKA_BOOTSTRAP_SERVER=broker:29092 - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 diff --git a/docker/quickstart/docker-compose.consumers.quickstart.yml b/docker/quickstart/docker-compose.consumers.quickstart.yml index 49c16932cc844..38501bd63808d 100644 --- a/docker/quickstart/docker-compose.consumers.quickstart.yml +++ b/docker/quickstart/docker-compose.consumers.quickstart.yml @@ -17,11 +17,7 @@ services: - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - NEO4J_HOST=http://neo4j:7474 - - NEO4J_URI=bolt://neo4j - - NEO4J_USERNAME=neo4j - - NEO4J_PASSWORD=datahub - - GRAPH_SERVICE_IMPL=neo4j + - GRAPH_SERVICE_IMPL=elasticsearch - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml hostname: datahub-mae-consumer image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head} @@ -29,14 +25,21 @@ services: - 9091:9091 datahub-mce-consumer: container_name: datahub-mce-consumer + depends_on: + - neo4j environment: - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - NEO4J_HOST=http://neo4j:7474 + - NEO4J_URI=bolt://neo4j + - NEO4J_USERNAME=neo4j + - NEO4J_PASSWORD=datahub + - GRAPH_SERVICE_IMPL=neo4j - MCE_CONSUMER_ENABLED=true - EBEAN_DATASOURCE_USERNAME=datahub - EBEAN_DATASOURCE_PASSWORD=datahub - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - KAFKA_BOOTSTRAP_SERVER=broker:29092 - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 @@ -44,7 +47,6 @@ services: - ELASTICSEARCH_PORT=9200 - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - GRAPH_SERVICE_IMPL=elasticsearch - JAVA_OPTS=-Xms1g -Xmx1g - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index e203b725c6687..8bf9588314b50 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -36,15 +36,6 @@ services: - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT - DATAHUB_ACTIONS_SLACK_ENABLED - - DATAHUB_ACTIONS_SLACK_DATAHUB_BASE_URL - - DATAHUB_ACTIONS_SLACK_BOT_TOKEN - - DATAHUB_ACTIONS_SLACK_SIGNING_SECRET - - DATAHUB_ACTIONS_SLACK_CHANNEL - - DATAHUB_ACTIONS_SLACK_SUPPRESS_SYSTEM_ACTIVITY - - DATAHUB_ACTIONS_TEAMS_ENABLED - - DATAHUB_ACTIONS_TEAMS_DATAHUB_BASE_URL - - DATAHUB_ACTIONS_TEAMS_WEBHOOK_URL - - DATAHUB_ACTIONS_TEAMS_SUPPRESS_SYSTEM_ACTIVITY hostname: actions image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} restart: on-failure:5 diff --git a/docker/quickstart/generate_docker_quickstart.py b/docker/quickstart/generate_docker_quickstart.py index 7d7ffacc8cfb6..33afcafed2751 100644 --- a/docker/quickstart/generate_docker_quickstart.py +++ b/docker/quickstart/generate_docker_quickstart.py @@ -116,6 +116,9 @@ def generate(compose_files, output_file) -> None: for modified_file in modified_files: dict_merge(merged_docker_config, modified_file) + # Dedup env vars, last wins + dedup_env_vars(merged_docker_config) + # Write output file output_dir = os.path.dirname(output_file) if len(output_dir) and not os.path.exists(output_dir): @@ -130,5 +133,23 @@ def generate(compose_files, output_file) -> None: print(f"Successfully generated {output_file}.") +def dedup_env_vars(merged_docker_config): + for service in merged_docker_config['services']: + if 'environment' in merged_docker_config['services'][service]: + lst = merged_docker_config['services'][service]['environment'] + if lst is not None: + # use a set to cache duplicates + caches = set() + results = [] + for item in lst: + prefix = item.rpartition('=')[0] + # check whether prefix already exists + if prefix not in caches: + results.append(item) + caches.add(prefix) + if set(lst) != set(results): + merged_docker_config['services'][service]['environment'] = results + + if __name__ == "__main__": generate() From 733d586961e43fe47cc5659ae3e264a8beff0257 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Thu, 19 Jan 2023 14:35:18 +0000 Subject: [PATCH 07/22] change mae consumer env var file --- docker/docker-compose.consumers.yml | 2 +- ...ocker-compose-without-neo4j.quickstart.yml | 32 +++++++++---------- .../docker-compose.consumers.quickstart.yml | 6 +++- .../quickstart/docker-compose.quickstart.yml | 2 +- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/docker/docker-compose.consumers.yml b/docker/docker-compose.consumers.yml index f7210be9c8f20..dcfdff49aa3cd 100644 --- a/docker/docker-compose.consumers.yml +++ b/docker/docker-compose.consumers.yml @@ -11,7 +11,7 @@ services: context: ../ dockerfile: docker/datahub-mae-consumer/Dockerfile image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head} - env_file: datahub-mae-consumer/env/docker-without-neo4j.env + env_file: datahub-mae-consumer/env/docker.env hostname: datahub-mae-consumer container_name: datahub-mae-consumer ports: diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 4f3be96a66955..f649382579d85 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -65,29 +65,29 @@ services: depends_on: - mysql environment: + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - GRAPH_SERVICE_IMPL=elasticsearch - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - EBEAN_DATASOURCE_HOST=mysql:3306 - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - JAVA_OPTS=-Xms1g -Xmx1g + - ELASTICSEARCH_PORT=9200 + - MCE_CONSUMER_ENABLED=true - PE_CONSUMER_ENABLED=true - - EBEAN_DATASOURCE_USERNAME=datahub - - EBEAN_DATASOURCE_PASSWORD=datahub + - ENTITY_SERVICE_ENABLE_RETENTION=true + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - UI_INGESTION_ENABLED=true - - ELASTICSEARCH_HOST=elasticsearch - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - JAVA_OPTS=-Xms1g -Xmx1g - - ENTITY_SERVICE_ENABLE_RETENTION=true + - EBEAN_DATASOURCE_HOST=mysql:3306 + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - MAE_CONSUMER_ENABLED=true - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - GRAPH_SERVICE_IMPL=elasticsearch + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - EBEAN_DATASOURCE_USERNAME=datahub + - EBEAN_DATASOURCE_PASSWORD=datahub - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - MAE_CONSUMER_ENABLED=true - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - ELASTICSEARCH_PORT=9200 - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - MCE_CONSUMER_ENABLED=true - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: diff --git a/docker/quickstart/docker-compose.consumers.quickstart.yml b/docker/quickstart/docker-compose.consumers.quickstart.yml index 38501bd63808d..8dbe776a183e2 100644 --- a/docker/quickstart/docker-compose.consumers.quickstart.yml +++ b/docker/quickstart/docker-compose.consumers.quickstart.yml @@ -17,7 +17,11 @@ services: - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - GRAPH_SERVICE_IMPL=elasticsearch + - NEO4J_HOST=http://neo4j:7474 + - NEO4J_URI=bolt://neo4j + - NEO4J_USERNAME=neo4j + - NEO4J_PASSWORD=datahub + - GRAPH_SERVICE_IMPL=neo4j - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml hostname: datahub-mae-consumer image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head} diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 8bf9588314b50..768159ebc4634 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -65,8 +65,8 @@ services: datahub-gms: container_name: datahub-gms depends_on: - - mysql - neo4j + - mysql environment: - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} From 91ae779995fdda9172878e44ef0b860158849348 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Thu, 19 Jan 2023 16:01:00 +0000 Subject: [PATCH 08/22] Fix compose check logic, adds neo4j supported m1 docker-compose --- .github/workflows/build-and-test.yml | 3 + .../docker-compose-without-neo4j.override.yml | 2 +- docker/docker-compose.override.yml | 2 +- .../docker-compose-m1.quickstart.yml | 204 ++++++++++++++++++ ...er-compose-without-neo4j-m1.quickstart.yml | 73 +++---- ...ocker-compose-without-neo4j.quickstart.yml | 37 ++-- .../quickstart/docker-compose.quickstart.yml | 3 +- docker/quickstart/generate_and_compare.sh | 5 +- .../quickstart/generate_docker_quickstart.py | 6 +- .../quickstart/generate_docker_quickstart.sh | 7 +- 10 files changed, 272 insertions(+), 70 deletions(-) create mode 100644 docker/quickstart/docker-compose-m1.quickstart.yml diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index dfe4b8354d435..2f30a4ab583ae 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -73,6 +73,9 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.7" + - name: Install yq for yaml parsing + run: VERSION=v4.30.8 and BINARY=yq_linux_amd64 wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY}.tar.gz -O - |\ + tar xz && mv ${BINARY} /usr/bin/yq - name: Quickstart Compose Validation run: ./docker/quickstart/generate_and_compare.sh diff --git a/docker/docker-compose-without-neo4j.override.yml b/docker/docker-compose-without-neo4j.override.yml index f09ab0ce08b1e..afaec5df0e665 100644 --- a/docker/docker-compose-without-neo4j.override.yml +++ b/docker/docker-compose-without-neo4j.override.yml @@ -6,7 +6,7 @@ services: hostname: mysql image: mysql:5.7 env_file: mysql/env/docker.env - command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password ports: - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 volumes: diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml index 4446c45483220..0b99c643781df 100644 --- a/docker/docker-compose.override.yml +++ b/docker/docker-compose.override.yml @@ -7,7 +7,7 @@ services: hostname: mysql image: mysql:5.7 env_file: mysql/env/docker.env - command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password ports: - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 volumes: diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml new file mode 100644 index 0000000000000..35b2ad47c2433 --- /dev/null +++ b/docker/quickstart/docker-compose-m1.quickstart.yml @@ -0,0 +1,204 @@ +networks: + default: + name: datahub_network +services: + broker: + container_name: broker + depends_on: + - zookeeper + environment: + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m + - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false + hostname: broker + image: kymeric/cp-kafka:latest + ports: + - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 + volumes: + - broker:/var/lib/kafka/data/ + datahub-actions: + depends_on: + - datahub-gms + environment: + - DATAHUB_GMS_PROTOCOL=http + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - SCHEMA_REGISTRY_URL=http://schema-registry:8081 + - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 + - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 + - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system + - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT + hostname: actions + image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} + restart: on-failure:5 + datahub-frontend-react: + container_name: datahub-frontend-react + depends_on: + - datahub-gms + environment: + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - DATAHUB_SECRET=YouKnowNothing + - DATAHUB_APP_VERSION=1.0 + - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB + - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf + -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml + -Dlogback.debug=false -Dpidfile.path=/dev/null + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 + - ELASTIC_CLIENT_HOST=elasticsearch + - ELASTIC_CLIENT_PORT=9200 + hostname: datahub-frontend-react + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} + ports: + - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 + volumes: + - ${HOME}/.datahub/plugins:/etc/datahub/plugins + datahub-gms: + container_name: datahub-gms + depends_on: + - mysql + - neo4j + environment: + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - EBEAN_DATASOURCE_USERNAME=datahub + - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - NEO4J_HOST=http://neo4j:7474 + - NEO4J_URI=bolt://neo4j + - NEO4J_USERNAME=neo4j + - NEO4J_PASSWORD=datahub + - JAVA_OPTS=-Xms1g -Xmx1g + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - GRAPH_SERVICE_IMPL=neo4j + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - ENTITY_SERVICE_ENABLE_RETENTION=true + - MAE_CONSUMER_ENABLED=true + - MCE_CONSUMER_ENABLED=true + - PE_CONSUMER_ENABLED=true + - UI_INGESTION_ENABLED=true + - METADATA_SERVICE_AUTH_ENABLED=false + hostname: datahub-gms + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} + ports: + - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 + volumes: + - ${HOME}/.datahub/plugins/:/etc/datahub/plugins + - ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources + elasticsearch: + container_name: elasticsearch + environment: + - discovery.type=single-node + - xpack.security.enabled=false + - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true + healthcheck: + retries: 4 + start_period: 2m + test: + - CMD-SHELL + - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' + || exit 1 + hostname: elasticsearch + image: elasticsearch:7.9.3 + mem_limit: 1g + ports: + - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 + volumes: + - esdata:/usr/share/elasticsearch/data + elasticsearch-setup: + container_name: elasticsearch-setup + depends_on: + - elasticsearch + environment: + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_PROTOCOL=http + hostname: elasticsearch-setup + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} + mysql: + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password + container_name: mysql + environment: + - MYSQL_DATABASE=datahub + - MYSQL_USER=datahub + - MYSQL_PASSWORD=datahub + - MYSQL_ROOT_PASSWORD=datahub + hostname: mysql + image: mariadb:10.5.8 + ports: + - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 + volumes: + - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql + - mysqldata:/var/lib/mysql + mysql-setup: + container_name: mysql-setup + depends_on: + - mysql + environment: + - MYSQL_HOST=mysql + - MYSQL_PORT=3306 + - MYSQL_USERNAME=datahub + - MYSQL_PASSWORD=datahub + - DATAHUB_DB_NAME=datahub + hostname: mysql-setup + image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} + neo4j: + container_name: neo4j + environment: + - NEO4J_AUTH=neo4j/datahub + - NEO4J_dbms_default__database=graph.db + - NEO4J_dbms_allow__upgrade=true + hostname: neo4j + image: neo4j/neo4j-arm64-experimental:4.0.6-arm64 + ports: + - ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474 + - ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687 + volumes: + - neo4jdata:/data + schema-registry: + container_name: schema-registry + depends_on: + - zookeeper + - broker + environment: + - SCHEMA_REGISTRY_HOST_NAME=schemaregistry + - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 + hostname: schema-registry + image: eugenetea/schema-registry-arm64:latest + ports: + - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 + zookeeper: + container_name: zookeeper + environment: + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 + hostname: zookeeper + image: kymeric/cp-zookeeper:latest + ports: + - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 + volumes: + - zkdata:/var/lib/zookeeper +version: '2.3' +volumes: + broker: null + esdata: null + mysqldata: null + neo4jdata: null + zkdata: null diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index bf29a265410f1..b73cc88435bbe 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -16,33 +16,23 @@ services: - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false hostname: broker - image: confluentinc/cp-kafka:7.2.2 + image: kymeric/cp-kafka:latest ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 datahub-actions: depends_on: - - datahub-gms + - datahub-gms environment: - - DATAHUB_GMS_PROTOCOL=http - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - SCHEMA_REGISTRY_URL=http://schema-registry:8081 - - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 - - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT - - DATAHUB_ACTIONS_SLACK_ENABLED - - DATAHUB_ACTIONS_SLACK_DATAHUB_BASE_URL - - DATAHUB_ACTIONS_SLACK_BOT_TOKEN - - DATAHUB_ACTIONS_SLACK_SIGNING_SECRET - - DATAHUB_ACTIONS_SLACK_CHANNEL - - DATAHUB_ACTIONS_SLACK_SUPPRESS_SYSTEM_ACTIVITY - - DATAHUB_ACTIONS_TEAMS_ENABLED - - DATAHUB_ACTIONS_TEAMS_DATAHUB_BASE_URL - - DATAHUB_ACTIONS_TEAMS_WEBHOOK_URL - - DATAHUB_ACTIONS_TEAMS_SUPPRESS_SYSTEM_ACTIVITY + - DATAHUB_GMS_PROTOCOL=http + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - SCHEMA_REGISTRY_URL=http://schema-registry:8081 + - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 + - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 + - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system + - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT hostname: actions image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} restart: on-failure:5 @@ -68,33 +58,35 @@ services: ports: - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/plugins:/etc/datahub/plugins datahub-gms: container_name: datahub-gms depends_on: - mysql environment: - - EBEAN_DATASOURCE_USERNAME=datahub - - EBEAN_DATASOURCE_PASSWORD=datahub - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - MAE_CONSUMER_ENABLED=true + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - UI_INGESTION_ENABLED=true + - ELASTICSEARCH_PORT=9200 + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 + - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - MCE_CONSUMER_ENABLED=true + - JAVA_OPTS=-Xms1g -Xmx1g + - PE_CONSUMER_ENABLED=true - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - ELASTICSEARCH_HOST=elasticsearch + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - EBEAN_DATASOURCE_USERNAME=datahub - GRAPH_SERVICE_IMPL=elasticsearch - - JAVA_OPTS=-Xms1g -Xmx1g - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - MAE_CONSUMER_ENABLED=true - - MCE_CONSUMER_ENABLED=true - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - PE_CONSUMER_ENABLED=true + - ENTITY_SERVICE_ENABLE_RETENTION=true + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: @@ -141,7 +133,6 @@ services: - MYSQL_ROOT_PASSWORD=datahub hostname: mysql image: mariadb:10.5.8 - # image: mysql:8 ports: - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 volumes: @@ -166,9 +157,9 @@ services: - broker environment: - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=PLAINTEXT://broker:29092 + - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 hostname: schema-registry - image: confluentinc/cp-schema-registry:7.2.0 + image: eugenetea/schema-registry-arm64:latest ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: @@ -177,7 +168,7 @@ services: - ZOOKEEPER_CLIENT_PORT=2181 - ZOOKEEPER_TICK_TIME=2000 hostname: zookeeper - image: confluentinc/cp-zookeeper:7.2.2 + image: kymeric/cp-zookeeper:latest ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index f649382579d85..0047afe68d1e1 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -33,7 +33,6 @@ services: - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT - - DATAHUB_ACTIONS_SLACK_ENABLED hostname: actions image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} restart: on-failure:5 @@ -65,29 +64,29 @@ services: depends_on: - mysql environment: - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - GRAPH_SERVICE_IMPL=elasticsearch - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - JAVA_OPTS=-Xms1g -Xmx1g - - ELASTICSEARCH_PORT=9200 + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - MCE_CONSUMER_ENABLED=true - - PE_CONSUMER_ENABLED=true - - ENTITY_SERVICE_ENABLE_RETENTION=true + - EBEAN_DATASOURCE_PASSWORD=datahub + - ELASTICSEARCH_HOST=elasticsearch + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - UI_INGESTION_ENABLED=true - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - EBEAN_DATASOURCE_HOST=mysql:3306 - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - MAE_CONSUMER_ENABLED=true - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - EBEAN_DATASOURCE_USERNAME=datahub - - EBEAN_DATASOURCE_PASSWORD=datahub - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch + - MAE_CONSUMER_ENABLED=true + - ENTITY_SERVICE_ENABLE_RETENTION=true + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - GRAPH_SERVICE_IMPL=elasticsearch + - UI_INGESTION_ENABLED=true + - JAVA_OPTS=-Xms1g -Xmx1g + - PE_CONSUMER_ENABLED=true + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - ELASTICSEARCH_PORT=9200 hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: @@ -125,7 +124,7 @@ services: hostname: elasticsearch-setup image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} mysql: - command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password container_name: mysql environment: - MYSQL_DATABASE=datahub diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 768159ebc4634..3998d7bb87418 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -35,7 +35,6 @@ services: - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT - - DATAHUB_ACTIONS_SLACK_ENABLED hostname: actions image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} restart: on-failure:5 @@ -134,7 +133,7 @@ services: hostname: elasticsearch-setup image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} mysql: - command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password container_name: mysql environment: - MYSQL_DATABASE=datahub diff --git a/docker/quickstart/generate_and_compare.sh b/docker/quickstart/generate_and_compare.sh index 4e7c26b2655a2..587c9d6ec3bf1 100755 --- a/docker/quickstart/generate_and_compare.sh +++ b/docker/quickstart/generate_and_compare.sh @@ -14,13 +14,16 @@ source venv/bin/activate pip install -r requirements.txt python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml temp.quickstart.yml python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml temp-without-neo4j.quickstart.yml +python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml ../docker-compose.m1.yml temp-m1.quickstart.yml +python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml ../docker-compose-without-neo4j.m1.yml temp-without-neo4j-m1.quickstart.yml python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml temp.monitoring.quickstart.yml python generate_docker_quickstart.py ../docker-compose.consumers.yml temp.consumers.quickstart.yml python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml temp.consumers-without-neo4j.quickstart.yml for flavour in "${FLAVOURS[@]}" do - if cmp docker-compose$flavour.quickstart.yml temp$flavour.quickstart.yml; then + + if cmp <(yq -i -P 'sort_keys(..)' docker-compose$flavour.quickstart.yml) <(yq -i -P 'sort_keys(..)' temp$flavour.quickstart.yml); then echo "docker-compose$flavour.quickstart.yml is up to date." else echo "docker-compose$flavour.quickstart.yml is out of date." diff --git a/docker/quickstart/generate_docker_quickstart.py b/docker/quickstart/generate_docker_quickstart.py index 33afcafed2751..5b31a8b7f6e00 100644 --- a/docker/quickstart/generate_docker_quickstart.py +++ b/docker/quickstart/generate_docker_quickstart.py @@ -142,9 +142,11 @@ def dedup_env_vars(merged_docker_config): caches = set() results = [] for item in lst: - prefix = item.rpartition('=')[0] + partitions = item.rpartition('=') + prefix = partitions[0] + suffix = partitions[1] # check whether prefix already exists - if prefix not in caches: + if prefix not in caches and suffix != "": results.append(item) caches.add(prefix) if set(lst) != set(results): diff --git a/docker/quickstart/generate_docker_quickstart.sh b/docker/quickstart/generate_docker_quickstart.sh index 49d2f28751622..3d8ac7de9aa4c 100755 --- a/docker/quickstart/generate_docker_quickstart.sh +++ b/docker/quickstart/generate_docker_quickstart.sh @@ -11,8 +11,9 @@ source venv/bin/activate pip install -r requirements.txt python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml docker-compose.quickstart.yml python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml docker-compose-without-neo4j.quickstart.yml +python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml ../docker-compose.m1.yml docker-compose-m1.quickstart.yml +python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml ../docker-compose-without-neo4j.m1.yml docker-compose-without-neo4j-m1.quickstart.yml python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml docker-compose.monitoring.quickstart.yml -python generate_docker_quickstart.py ../docker-compose.consumers.yml docker-compose.consumers.quickstart.yml # Investigate -python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml docker-compose.consumers-without-neo4j.quickstart.yml # investigate +python generate_docker_quickstart.py ../docker-compose.consumers.yml docker-compose.consumers.quickstart.yml +python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml docker-compose.consumers-without-neo4j.quickstart.yml python generate_docker_quickstart.py ../docker-compose.kafka-setup.yml docker-compose.kafka-setup.quickstart.yml -# TODO docker-compose-without-neo4j-m1.quickstart.yml is not getting generated \ No newline at end of file From a6469f9ff5333fab5aedd80f6a2fee01fc192dd5 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Thu, 19 Jan 2023 16:14:34 +0000 Subject: [PATCH 09/22] Add support for Neo4J M1 quickstart --- .../docker-compose-m1.quickstart.yml | 2 +- ...er-compose-without-neo4j-m1.quickstart.yml | 32 +-- ...ocker-compose-without-neo4j.quickstart.yml | 191 ++++++++------- .../docker-compose.monitoring.quickstart.yml | 50 ++-- .../quickstart/docker-compose.quickstart.yml | 219 +++++++++--------- docker/quickstart/temp-m1.quickstart.yml | 204 ++++++++++++++++ .../temp-without-neo4j-m1.quickstart.yml | 180 ++++++++++++++ .../temp-without-neo4j.quickstart.yml | 177 ++++++++++++++ ...emp.consumers-without-neo4j.quickstart.yml | 54 +++++ .../quickstart/temp.consumers.quickstart.yml | 66 ++++++ .../quickstart/temp.monitoring.quickstart.yml | 47 ++++ docker/quickstart/temp.quickstart.yml | 201 ++++++++++++++++ .../src/datahub/cli/docker_cli.py | 27 ++- 13 files changed, 1190 insertions(+), 260 deletions(-) create mode 100644 docker/quickstart/temp-m1.quickstart.yml create mode 100644 docker/quickstart/temp-without-neo4j-m1.quickstart.yml create mode 100644 docker/quickstart/temp-without-neo4j.quickstart.yml create mode 100644 docker/quickstart/temp.consumers-without-neo4j.quickstart.yml create mode 100644 docker/quickstart/temp.consumers.quickstart.yml create mode 100644 docker/quickstart/temp.monitoring.quickstart.yml create mode 100644 docker/quickstart/temp.quickstart.yml diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml index 35b2ad47c2433..d19c68ac526a0 100644 --- a/docker/quickstart/docker-compose-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-m1.quickstart.yml @@ -64,8 +64,8 @@ services: datahub-gms: container_name: datahub-gms depends_on: - - mysql - neo4j + - mysql environment: - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index b73cc88435bbe..8359393381aec 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -64,29 +64,29 @@ services: depends_on: - mysql environment: - - EBEAN_DATASOURCE_HOST=mysql:3306 - - MAE_CONSUMER_ENABLED=true + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - GRAPH_SERVICE_IMPL=elasticsearch + - EBEAN_DATASOURCE_PASSWORD=datahub + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - EBEAN_DATASOURCE_USERNAME=datahub + - MAE_CONSUMER_ENABLED=true + - JAVA_OPTS=-Xms1g -Xmx1g + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - UI_INGESTION_ENABLED=true - ELASTICSEARCH_PORT=9200 + - MCE_CONSUMER_ENABLED=true - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_HOST=mysql:3306 + - ENTITY_SERVICE_ENABLE_RETENTION=true + - ELASTICSEARCH_HOST=elasticsearch + - UI_INGESTION_ENABLED=true + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - MCE_CONSUMER_ENABLED=true - - JAVA_OPTS=-Xms1g -Xmx1g - PE_CONSUMER_ENABLED=true - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - ELASTICSEARCH_HOST=elasticsearch - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - EBEAN_DATASOURCE_USERNAME=datahub - - GRAPH_SERVICE_IMPL=elasticsearch - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - ENTITY_SERVICE_ENABLE_RETENTION=true - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 0047afe68d1e1..064f21a32c59e 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -5,175 +5,172 @@ services: broker: container_name: broker depends_on: - - zookeeper + - zookeeper environment: - - KAFKA_BROKER_ID=1 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m + - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false hostname: broker image: confluentinc/cp-kafka:5.4.0 ports: - - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 + - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 datahub-actions: depends_on: - - datahub-gms + - datahub-gms environment: - - DATAHUB_GMS_PROTOCOL=http - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - SCHEMA_REGISTRY_URL=http://schema-registry:8081 - - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 - - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT + - DATAHUB_GMS_PROTOCOL=http + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - SCHEMA_REGISTRY_URL=http://schema-registry:8081 + - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 + - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 + - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system + - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT hostname: actions image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} restart: on-failure:5 datahub-frontend-react: container_name: datahub-frontend-react depends_on: - - datahub-gms + - datahub-gms environment: - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - DATAHUB_SECRET=YouKnowNothing - - DATAHUB_APP_VERSION=1.0 - - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf - -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml - -Dlogback.debug=false -Dpidfile.path=/dev/null - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - - ELASTIC_CLIENT_HOST=elasticsearch - - ELASTIC_CLIENT_PORT=9200 + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - DATAHUB_SECRET=YouKnowNothing + - DATAHUB_APP_VERSION=1.0 + - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB + - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 + - ELASTIC_CLIENT_HOST=elasticsearch + - ELASTIC_CLIENT_PORT=9200 hostname: datahub-frontend-react image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} ports: - - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 + - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/plugins:/etc/datahub/plugins datahub-gms: container_name: datahub-gms depends_on: - - mysql + - mysql environment: - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - MCE_CONSUMER_ENABLED=true - - EBEAN_DATASOURCE_PASSWORD=datahub - - ELASTICSEARCH_HOST=elasticsearch - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - EBEAN_DATASOURCE_HOST=mysql:3306 - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - EBEAN_DATASOURCE_USERNAME=datahub - - MAE_CONSUMER_ENABLED=true - - ENTITY_SERVICE_ENABLE_RETENTION=true - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - GRAPH_SERVICE_IMPL=elasticsearch - - UI_INGESTION_ENABLED=true - - JAVA_OPTS=-Xms1g -Xmx1g - - PE_CONSUMER_ENABLED=true - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - MAE_CONSUMER_ENABLED=true + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - EBEAN_DATASOURCE_USERNAME=datahub + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - MCE_CONSUMER_ENABLED=true + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - JAVA_OPTS=-Xms1g -Xmx1g + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - PE_CONSUMER_ENABLED=true + - EBEAN_DATASOURCE_HOST=mysql:3306 + - GRAPH_SERVICE_IMPL=elasticsearch + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - EBEAN_DATASOURCE_PASSWORD=datahub + - ENTITY_SERVICE_ENABLE_RETENTION=true + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - ELASTICSEARCH_HOST=elasticsearch + - UI_INGESTION_ENABLED=true + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - KAFKA_BOOTSTRAP_SERVER=broker:29092 hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: - - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 + - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/plugins:/etc/datahub/plugins elasticsearch: container_name: elasticsearch environment: - - discovery.type=single-node - - xpack.security.enabled=false - - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true + - discovery.type=single-node + - xpack.security.enabled=false + - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true healthcheck: retries: 4 start_period: 2m test: - - CMD-SHELL - - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' - || exit 1 + - CMD-SHELL + - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1 hostname: elasticsearch image: elasticsearch:7.9.3 mem_limit: 1g ports: - - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 + - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: - - esdata:/usr/share/elasticsearch/data + - esdata:/usr/share/elasticsearch/data elasticsearch-setup: container_name: elasticsearch-setup depends_on: - - elasticsearch + - elasticsearch environment: - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_PROTOCOL=http + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_PROTOCOL=http hostname: elasticsearch-setup image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} mysql: command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password container_name: mysql environment: - - MYSQL_DATABASE=datahub - - MYSQL_USER=datahub - - MYSQL_PASSWORD=datahub - - MYSQL_ROOT_PASSWORD=datahub + - MYSQL_DATABASE=datahub + - MYSQL_USER=datahub + - MYSQL_PASSWORD=datahub + - MYSQL_ROOT_PASSWORD=datahub hostname: mysql image: mysql:5.7 ports: - - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 + - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 volumes: - - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - - mysqldata:/var/lib/mysql + - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql + - mysqldata:/var/lib/mysql mysql-setup: container_name: mysql-setup depends_on: - - mysql + - mysql environment: - - MYSQL_HOST=mysql - - MYSQL_PORT=3306 - - MYSQL_USERNAME=datahub - - MYSQL_PASSWORD=datahub - - DATAHUB_DB_NAME=datahub + - MYSQL_HOST=mysql + - MYSQL_PORT=3306 + - MYSQL_USERNAME=datahub + - MYSQL_PASSWORD=datahub + - DATAHUB_DB_NAME=datahub hostname: mysql-setup image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} schema-registry: container_name: schema-registry depends_on: - - zookeeper - - broker + - zookeeper + - broker environment: - - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 + - SCHEMA_REGISTRY_HOST_NAME=schemaregistry + - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 hostname: schema-registry image: confluentinc/cp-schema-registry:5.4.0 ports: - - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 + - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: container_name: zookeeper environment: - - ZOOKEEPER_CLIENT_PORT=2181 - - ZOOKEEPER_TICK_TIME=2000 + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 hostname: zookeeper image: confluentinc/cp-zookeeper:5.4.0 ports: - - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 + - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: - - zkdata:/var/lib/zookeeper -version: '2.3' + - zkdata:/var/lib/zookeeper +version: "2.3" volumes: esdata: null mysqldata: null diff --git a/docker/quickstart/docker-compose.monitoring.quickstart.yml b/docker/quickstart/docker-compose.monitoring.quickstart.yml index 0796692605719..37ccd057a00ed 100644 --- a/docker/quickstart/docker-compose.monitoring.quickstart.yml +++ b/docker/quickstart/docker-compose.monitoring.quickstart.yml @@ -1,47 +1,47 @@ services: datahub-frontend-react: environment: - - ENABLE_PROMETHEUS=true - - ENABLE_OTEL=true - - OTEL_TRACES_EXPORTER=jaeger - - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250 - - OTEL_METRICS_EXPORTER=none - - OTEL_SERVICE_NAME=datahub-gms + - ENABLE_PROMETHEUS=true + - ENABLE_OTEL=true + - OTEL_TRACES_EXPORTER=jaeger + - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250 + - OTEL_METRICS_EXPORTER=none + - OTEL_SERVICE_NAME=datahub-gms ports: - - '4318' + - "4318" datahub-gms: environment: - - ENABLE_PROMETHEUS=true - - ENABLE_OTEL=true - - OTEL_TRACES_EXPORTER=jaeger - - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250 - - OTEL_METRICS_EXPORTER=none - - OTEL_SERVICE_NAME=datahub-gms + - ENABLE_PROMETHEUS=true + - ENABLE_OTEL=true + - OTEL_TRACES_EXPORTER=jaeger + - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250 + - OTEL_METRICS_EXPORTER=none + - OTEL_SERVICE_NAME=datahub-gms ports: - - '4318' + - "4318" grafana: depends_on: - - prometheus + - prometheus image: grafana/grafana:9.1.4 ports: - - 3001:3000 + - 3001:3000 volumes: - - grafana-storage:/var/lib/grafana - - ../monitoring/grafana/datasources:/etc/grafana/provisioning/datasources - - ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards + - grafana-storage:/var/lib/grafana + - ../monitoring/grafana/datasources:/etc/grafana/provisioning/datasources + - ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards jaeger-all-in-one: image: jaegertracing/all-in-one:latest ports: - - 16686:16686 - - '14268' - - '14250' + - 16686:16686 + - "14268" + - "14250" prometheus: container_name: prometheus image: prom/prometheus:latest ports: - - 9089:9090 + - 9089:9090 volumes: - - ../monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml -version: '2.3' + - ../monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml +version: "2.3" volumes: grafana-storage: null diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 3998d7bb87418..4b3557c8dae09 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -5,197 +5,194 @@ services: broker: container_name: broker depends_on: - - zookeeper + - zookeeper environment: - - KAFKA_BROKER_ID=1 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m + - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false hostname: broker image: confluentinc/cp-kafka:5.4.0 ports: - - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 + - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: - - broker:/var/lib/kafka/data/ + - broker:/var/lib/kafka/data/ datahub-actions: depends_on: - - datahub-gms + - datahub-gms environment: - - DATAHUB_GMS_PROTOCOL=http - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - SCHEMA_REGISTRY_URL=http://schema-registry:8081 - - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 - - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT + - DATAHUB_GMS_PROTOCOL=http + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - SCHEMA_REGISTRY_URL=http://schema-registry:8081 + - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 + - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 + - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system + - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT hostname: actions image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} restart: on-failure:5 datahub-frontend-react: container_name: datahub-frontend-react depends_on: - - datahub-gms + - datahub-gms environment: - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - DATAHUB_SECRET=YouKnowNothing - - DATAHUB_APP_VERSION=1.0 - - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf - -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml - -Dlogback.debug=false -Dpidfile.path=/dev/null - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - - ELASTIC_CLIENT_HOST=elasticsearch - - ELASTIC_CLIENT_PORT=9200 + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - DATAHUB_SECRET=YouKnowNothing + - DATAHUB_APP_VERSION=1.0 + - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB + - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 + - ELASTIC_CLIENT_HOST=elasticsearch + - ELASTIC_CLIENT_PORT=9200 hostname: datahub-frontend-react image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} ports: - - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 + - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/plugins:/etc/datahub/plugins datahub-gms: container_name: datahub-gms depends_on: - - neo4j - - mysql + - neo4j + - mysql environment: - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - EBEAN_DATASOURCE_USERNAME=datahub - - EBEAN_DATASOURCE_PASSWORD=datahub - - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - NEO4J_HOST=http://neo4j:7474 - - NEO4J_URI=bolt://neo4j - - NEO4J_USERNAME=neo4j - - NEO4J_PASSWORD=datahub - - JAVA_OPTS=-Xms1g -Xmx1g - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - GRAPH_SERVICE_IMPL=neo4j - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - ENTITY_SERVICE_ENABLE_RETENTION=true - - MAE_CONSUMER_ENABLED=true - - MCE_CONSUMER_ENABLED=true - - PE_CONSUMER_ENABLED=true - - UI_INGESTION_ENABLED=true - - METADATA_SERVICE_AUTH_ENABLED=false + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - EBEAN_DATASOURCE_USERNAME=datahub + - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - NEO4J_HOST=http://neo4j:7474 + - NEO4J_URI=bolt://neo4j + - NEO4J_USERNAME=neo4j + - NEO4J_PASSWORD=datahub + - JAVA_OPTS=-Xms1g -Xmx1g + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - GRAPH_SERVICE_IMPL=neo4j + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - ENTITY_SERVICE_ENABLE_RETENTION=true + - MAE_CONSUMER_ENABLED=true + - MCE_CONSUMER_ENABLED=true + - PE_CONSUMER_ENABLED=true + - UI_INGESTION_ENABLED=true + - METADATA_SERVICE_AUTH_ENABLED=false hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: - - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 + - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 volumes: - - ${HOME}/.datahub/plugins/:/etc/datahub/plugins - - ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources + - ${HOME}/.datahub/plugins/:/etc/datahub/plugins + - ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources elasticsearch: container_name: elasticsearch environment: - - discovery.type=single-node - - xpack.security.enabled=false - - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true + - discovery.type=single-node + - xpack.security.enabled=false + - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true healthcheck: retries: 4 start_period: 2m test: - - CMD-SHELL - - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' - || exit 1 + - CMD-SHELL + - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1 hostname: elasticsearch image: elasticsearch:7.9.3 mem_limit: 1g ports: - - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 + - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: - - esdata:/usr/share/elasticsearch/data + - esdata:/usr/share/elasticsearch/data elasticsearch-setup: container_name: elasticsearch-setup depends_on: - - elasticsearch + - elasticsearch environment: - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_PROTOCOL=http + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_PROTOCOL=http hostname: elasticsearch-setup image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} mysql: command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password container_name: mysql environment: - - MYSQL_DATABASE=datahub - - MYSQL_USER=datahub - - MYSQL_PASSWORD=datahub - - MYSQL_ROOT_PASSWORD=datahub + - MYSQL_DATABASE=datahub + - MYSQL_USER=datahub + - MYSQL_PASSWORD=datahub + - MYSQL_ROOT_PASSWORD=datahub hostname: mysql image: mysql:5.7 ports: - - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 + - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 volumes: - - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - - mysqldata:/var/lib/mysql + - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql + - mysqldata:/var/lib/mysql mysql-setup: container_name: mysql-setup depends_on: - - mysql + - mysql environment: - - MYSQL_HOST=mysql - - MYSQL_PORT=3306 - - MYSQL_USERNAME=datahub - - MYSQL_PASSWORD=datahub - - DATAHUB_DB_NAME=datahub + - MYSQL_HOST=mysql + - MYSQL_PORT=3306 + - MYSQL_USERNAME=datahub + - MYSQL_PASSWORD=datahub + - DATAHUB_DB_NAME=datahub hostname: mysql-setup image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} neo4j: container_name: neo4j environment: - - NEO4J_AUTH=neo4j/datahub - - NEO4J_dbms_default__database=graph.db - - NEO4J_dbms_allow__upgrade=true + - NEO4J_AUTH=neo4j/datahub + - NEO4J_dbms_default__database=graph.db + - NEO4J_dbms_allow__upgrade=true hostname: neo4j image: neo4j:4.4.9-community ports: - - ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474 - - ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687 + - ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474 + - ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687 volumes: - - neo4jdata:/data + - neo4jdata:/data schema-registry: container_name: schema-registry depends_on: - - zookeeper - - broker + - zookeeper + - broker environment: - - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 + - SCHEMA_REGISTRY_HOST_NAME=schemaregistry + - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 hostname: schema-registry image: confluentinc/cp-schema-registry:5.4.0 ports: - - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 + - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: container_name: zookeeper environment: - - ZOOKEEPER_CLIENT_PORT=2181 - - ZOOKEEPER_TICK_TIME=2000 + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 hostname: zookeeper image: confluentinc/cp-zookeeper:5.4.0 ports: - - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 + - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: - - zkdata:/var/lib/zookeeper -version: '2.3' + - zkdata:/var/lib/zookeeper +version: "2.3" volumes: broker: null esdata: null diff --git a/docker/quickstart/temp-m1.quickstart.yml b/docker/quickstart/temp-m1.quickstart.yml new file mode 100644 index 0000000000000..d19c68ac526a0 --- /dev/null +++ b/docker/quickstart/temp-m1.quickstart.yml @@ -0,0 +1,204 @@ +networks: + default: + name: datahub_network +services: + broker: + container_name: broker + depends_on: + - zookeeper + environment: + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m + - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false + hostname: broker + image: kymeric/cp-kafka:latest + ports: + - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 + volumes: + - broker:/var/lib/kafka/data/ + datahub-actions: + depends_on: + - datahub-gms + environment: + - DATAHUB_GMS_PROTOCOL=http + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - SCHEMA_REGISTRY_URL=http://schema-registry:8081 + - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 + - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 + - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system + - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT + hostname: actions + image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} + restart: on-failure:5 + datahub-frontend-react: + container_name: datahub-frontend-react + depends_on: + - datahub-gms + environment: + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - DATAHUB_SECRET=YouKnowNothing + - DATAHUB_APP_VERSION=1.0 + - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB + - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf + -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml + -Dlogback.debug=false -Dpidfile.path=/dev/null + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 + - ELASTIC_CLIENT_HOST=elasticsearch + - ELASTIC_CLIENT_PORT=9200 + hostname: datahub-frontend-react + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} + ports: + - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 + volumes: + - ${HOME}/.datahub/plugins:/etc/datahub/plugins + datahub-gms: + container_name: datahub-gms + depends_on: + - neo4j + - mysql + environment: + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - EBEAN_DATASOURCE_USERNAME=datahub + - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - NEO4J_HOST=http://neo4j:7474 + - NEO4J_URI=bolt://neo4j + - NEO4J_USERNAME=neo4j + - NEO4J_PASSWORD=datahub + - JAVA_OPTS=-Xms1g -Xmx1g + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - GRAPH_SERVICE_IMPL=neo4j + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - ENTITY_SERVICE_ENABLE_RETENTION=true + - MAE_CONSUMER_ENABLED=true + - MCE_CONSUMER_ENABLED=true + - PE_CONSUMER_ENABLED=true + - UI_INGESTION_ENABLED=true + - METADATA_SERVICE_AUTH_ENABLED=false + hostname: datahub-gms + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} + ports: + - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 + volumes: + - ${HOME}/.datahub/plugins/:/etc/datahub/plugins + - ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources + elasticsearch: + container_name: elasticsearch + environment: + - discovery.type=single-node + - xpack.security.enabled=false + - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true + healthcheck: + retries: 4 + start_period: 2m + test: + - CMD-SHELL + - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' + || exit 1 + hostname: elasticsearch + image: elasticsearch:7.9.3 + mem_limit: 1g + ports: + - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 + volumes: + - esdata:/usr/share/elasticsearch/data + elasticsearch-setup: + container_name: elasticsearch-setup + depends_on: + - elasticsearch + environment: + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_PROTOCOL=http + hostname: elasticsearch-setup + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} + mysql: + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password + container_name: mysql + environment: + - MYSQL_DATABASE=datahub + - MYSQL_USER=datahub + - MYSQL_PASSWORD=datahub + - MYSQL_ROOT_PASSWORD=datahub + hostname: mysql + image: mariadb:10.5.8 + ports: + - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 + volumes: + - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql + - mysqldata:/var/lib/mysql + mysql-setup: + container_name: mysql-setup + depends_on: + - mysql + environment: + - MYSQL_HOST=mysql + - MYSQL_PORT=3306 + - MYSQL_USERNAME=datahub + - MYSQL_PASSWORD=datahub + - DATAHUB_DB_NAME=datahub + hostname: mysql-setup + image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} + neo4j: + container_name: neo4j + environment: + - NEO4J_AUTH=neo4j/datahub + - NEO4J_dbms_default__database=graph.db + - NEO4J_dbms_allow__upgrade=true + hostname: neo4j + image: neo4j/neo4j-arm64-experimental:4.0.6-arm64 + ports: + - ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474 + - ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687 + volumes: + - neo4jdata:/data + schema-registry: + container_name: schema-registry + depends_on: + - zookeeper + - broker + environment: + - SCHEMA_REGISTRY_HOST_NAME=schemaregistry + - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 + hostname: schema-registry + image: eugenetea/schema-registry-arm64:latest + ports: + - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 + zookeeper: + container_name: zookeeper + environment: + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 + hostname: zookeeper + image: kymeric/cp-zookeeper:latest + ports: + - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 + volumes: + - zkdata:/var/lib/zookeeper +version: '2.3' +volumes: + broker: null + esdata: null + mysqldata: null + neo4jdata: null + zkdata: null diff --git a/docker/quickstart/temp-without-neo4j-m1.quickstart.yml b/docker/quickstart/temp-without-neo4j-m1.quickstart.yml new file mode 100644 index 0000000000000..06587b31fbe33 --- /dev/null +++ b/docker/quickstart/temp-without-neo4j-m1.quickstart.yml @@ -0,0 +1,180 @@ +networks: + default: + name: datahub_network +services: + broker: + container_name: broker + depends_on: + - zookeeper + environment: + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m + - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false + hostname: broker + image: kymeric/cp-kafka:latest + ports: + - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 + datahub-actions: + depends_on: + - datahub-gms + environment: + - DATAHUB_GMS_PROTOCOL=http + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - SCHEMA_REGISTRY_URL=http://schema-registry:8081 + - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 + - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 + - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system + - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT + hostname: actions + image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} + restart: on-failure:5 + datahub-frontend-react: + container_name: datahub-frontend-react + depends_on: + - datahub-gms + environment: + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - DATAHUB_SECRET=YouKnowNothing + - DATAHUB_APP_VERSION=1.0 + - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB + - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf + -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml + -Dlogback.debug=false -Dpidfile.path=/dev/null + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 + - ELASTIC_CLIENT_HOST=elasticsearch + - ELASTIC_CLIENT_PORT=9200 + hostname: datahub-frontend-react + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} + ports: + - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 + volumes: + - ${HOME}/.datahub/plugins:/etc/datahub/plugins + datahub-gms: + container_name: datahub-gms + depends_on: + - mysql + environment: + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - JAVA_OPTS=-Xms1g -Xmx1g + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - MCE_CONSUMER_ENABLED=true + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - UI_INGESTION_ENABLED=true + - MAE_CONSUMER_ENABLED=true + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_HOST=elasticsearch + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - EBEAN_DATASOURCE_PASSWORD=datahub + - PE_CONSUMER_ENABLED=true + - ENTITY_SERVICE_ENABLE_RETENTION=true + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - GRAPH_SERVICE_IMPL=elasticsearch + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_USERNAME=datahub + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + hostname: datahub-gms + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} + ports: + - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 + volumes: + - ${HOME}/.datahub/plugins:/etc/datahub/plugins + elasticsearch: + container_name: elasticsearch + environment: + - discovery.type=single-node + - xpack.security.enabled=false + - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true + healthcheck: + retries: 4 + start_period: 2m + test: + - CMD-SHELL + - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' + || exit 1 + hostname: elasticsearch + image: elasticsearch:7.9.3 + mem_limit: 1g + ports: + - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 + volumes: + - esdata:/usr/share/elasticsearch/data + elasticsearch-setup: + container_name: elasticsearch-setup + depends_on: + - elasticsearch + environment: + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_PROTOCOL=http + hostname: elasticsearch-setup + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} + mysql: + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password + container_name: mysql + environment: + - MYSQL_DATABASE=datahub + - MYSQL_USER=datahub + - MYSQL_PASSWORD=datahub + - MYSQL_ROOT_PASSWORD=datahub + hostname: mysql + image: mariadb:10.5.8 + ports: + - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 + volumes: + - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql + - mysqldata:/var/lib/mysql + mysql-setup: + container_name: mysql-setup + depends_on: + - mysql + environment: + - MYSQL_HOST=mysql + - MYSQL_PORT=3306 + - MYSQL_USERNAME=datahub + - MYSQL_PASSWORD=datahub + - DATAHUB_DB_NAME=datahub + hostname: mysql-setup + image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} + schema-registry: + container_name: schema-registry + depends_on: + - zookeeper + - broker + environment: + - SCHEMA_REGISTRY_HOST_NAME=schemaregistry + - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 + hostname: schema-registry + image: eugenetea/schema-registry-arm64:latest + ports: + - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 + zookeeper: + container_name: zookeeper + environment: + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 + hostname: zookeeper + image: kymeric/cp-zookeeper:latest + ports: + - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 + volumes: + - zkdata:/var/lib/zookeeper +version: '2.3' +volumes: + esdata: null + mysqldata: null + zkdata: null diff --git a/docker/quickstart/temp-without-neo4j.quickstart.yml b/docker/quickstart/temp-without-neo4j.quickstart.yml new file mode 100644 index 0000000000000..d388197d8b04f --- /dev/null +++ b/docker/quickstart/temp-without-neo4j.quickstart.yml @@ -0,0 +1,177 @@ +networks: + default: + name: datahub_network +services: + broker: + container_name: broker + depends_on: + - zookeeper + environment: + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m + - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false + hostname: broker + image: confluentinc/cp-kafka:5.4.0 + ports: + - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 + datahub-actions: + depends_on: + - datahub-gms + environment: + - DATAHUB_GMS_PROTOCOL=http + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - SCHEMA_REGISTRY_URL=http://schema-registry:8081 + - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 + - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 + - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system + - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT + hostname: actions + image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} + restart: on-failure:5 + datahub-frontend-react: + container_name: datahub-frontend-react + depends_on: + - datahub-gms + environment: + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - DATAHUB_SECRET=YouKnowNothing + - DATAHUB_APP_VERSION=1.0 + - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB + - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 + - ELASTIC_CLIENT_HOST=elasticsearch + - ELASTIC_CLIENT_PORT=9200 + hostname: datahub-frontend-react + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} + ports: + - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 + volumes: + - ${HOME}/.datahub/plugins:/etc/datahub/plugins + datahub-gms: + container_name: datahub-gms + depends_on: + - mysql + environment: + - EBEAN_DATASOURCE_USERNAME=datahub + - ELASTICSEARCH_PORT=9200 + - GRAPH_SERVICE_IMPL=elasticsearch + - UI_INGESTION_ENABLED=true + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - MCE_CONSUMER_ENABLED=true + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - JAVA_OPTS=-Xms1g -Xmx1g + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - EBEAN_DATASOURCE_HOST=mysql:3306 + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - MAE_CONSUMER_ENABLED=true + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - PE_CONSUMER_ENABLED=true + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - ENTITY_SERVICE_ENABLE_RETENTION=true + - EBEAN_DATASOURCE_PASSWORD=datahub + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + hostname: datahub-gms + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} + ports: + - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 + volumes: + - ${HOME}/.datahub/plugins:/etc/datahub/plugins + elasticsearch: + container_name: elasticsearch + environment: + - discovery.type=single-node + - xpack.security.enabled=false + - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true + healthcheck: + retries: 4 + start_period: 2m + test: + - CMD-SHELL + - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1 + hostname: elasticsearch + image: elasticsearch:7.9.3 + mem_limit: 1g + ports: + - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 + volumes: + - esdata:/usr/share/elasticsearch/data + elasticsearch-setup: + container_name: elasticsearch-setup + depends_on: + - elasticsearch + environment: + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_PROTOCOL=http + hostname: elasticsearch-setup + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} + mysql: + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password + container_name: mysql + environment: + - MYSQL_DATABASE=datahub + - MYSQL_USER=datahub + - MYSQL_PASSWORD=datahub + - MYSQL_ROOT_PASSWORD=datahub + hostname: mysql + image: mysql:5.7 + ports: + - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 + volumes: + - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql + - mysqldata:/var/lib/mysql + mysql-setup: + container_name: mysql-setup + depends_on: + - mysql + environment: + - MYSQL_HOST=mysql + - MYSQL_PORT=3306 + - MYSQL_USERNAME=datahub + - MYSQL_PASSWORD=datahub + - DATAHUB_DB_NAME=datahub + hostname: mysql-setup + image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} + schema-registry: + container_name: schema-registry + depends_on: + - zookeeper + - broker + environment: + - SCHEMA_REGISTRY_HOST_NAME=schemaregistry + - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 + hostname: schema-registry + image: confluentinc/cp-schema-registry:5.4.0 + ports: + - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 + zookeeper: + container_name: zookeeper + environment: + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 + hostname: zookeeper + image: confluentinc/cp-zookeeper:5.4.0 + ports: + - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 + volumes: + - zkdata:/var/lib/zookeeper +version: "2.3" +volumes: + esdata: null + mysqldata: null + zkdata: null diff --git a/docker/quickstart/temp.consumers-without-neo4j.quickstart.yml b/docker/quickstart/temp.consumers-without-neo4j.quickstart.yml new file mode 100644 index 0000000000000..cee66989456e7 --- /dev/null +++ b/docker/quickstart/temp.consumers-without-neo4j.quickstart.yml @@ -0,0 +1,54 @@ +services: + datahub-gms: + environment: + - MAE_CONSUMER_ENABLED=false + - MCE_CONSUMER_ENABLED=false + datahub-mae-consumer: + container_name: datahub-mae-consumer + environment: + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - MAE_CONSUMER_ENABLED=true + - PE_CONSUMER_ENABLED=true + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - GRAPH_SERVICE_IMPL=elasticsearch + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml + hostname: datahub-mae-consumer + image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head} + ports: + - 9091:9091 + datahub-mce-consumer: + container_name: datahub-mce-consumer + environment: + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - MCE_CONSUMER_ENABLED=true + - EBEAN_DATASOURCE_USERNAME=datahub + - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - GRAPH_SERVICE_IMPL=elasticsearch + - JAVA_OPTS=-Xms1g -Xmx1g + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml + - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system + - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - ENTITY_SERVICE_ENABLE_RETENTION=true + - MAE_CONSUMER_ENABLED=false + - PE_CONSUMER_ENABLED=false + - UI_INGESTION_ENABLED=false + hostname: datahub-mce-consumer + image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head} + ports: + - 9090:9090 +version: '2.3' diff --git a/docker/quickstart/temp.consumers.quickstart.yml b/docker/quickstart/temp.consumers.quickstart.yml new file mode 100644 index 0000000000000..8dbe776a183e2 --- /dev/null +++ b/docker/quickstart/temp.consumers.quickstart.yml @@ -0,0 +1,66 @@ +services: + datahub-gms: + environment: + - MAE_CONSUMER_ENABLED=false + - MCE_CONSUMER_ENABLED=false + datahub-mae-consumer: + container_name: datahub-mae-consumer + depends_on: + - neo4j + environment: + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - MAE_CONSUMER_ENABLED=true + - PE_CONSUMER_ENABLED=true + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - NEO4J_HOST=http://neo4j:7474 + - NEO4J_URI=bolt://neo4j + - NEO4J_USERNAME=neo4j + - NEO4J_PASSWORD=datahub + - GRAPH_SERVICE_IMPL=neo4j + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml + hostname: datahub-mae-consumer + image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head} + ports: + - 9091:9091 + datahub-mce-consumer: + container_name: datahub-mce-consumer + depends_on: + - neo4j + environment: + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - NEO4J_HOST=http://neo4j:7474 + - NEO4J_URI=bolt://neo4j + - NEO4J_USERNAME=neo4j + - NEO4J_PASSWORD=datahub + - GRAPH_SERVICE_IMPL=neo4j + - MCE_CONSUMER_ENABLED=true + - EBEAN_DATASOURCE_USERNAME=datahub + - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - JAVA_OPTS=-Xms1g -Xmx1g + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml + - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system + - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - ENTITY_SERVICE_ENABLE_RETENTION=true + - MAE_CONSUMER_ENABLED=false + - PE_CONSUMER_ENABLED=false + - UI_INGESTION_ENABLED=false + hostname: datahub-mce-consumer + image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head} + ports: + - 9090:9090 +version: '2.3' diff --git a/docker/quickstart/temp.monitoring.quickstart.yml b/docker/quickstart/temp.monitoring.quickstart.yml new file mode 100644 index 0000000000000..37ccd057a00ed --- /dev/null +++ b/docker/quickstart/temp.monitoring.quickstart.yml @@ -0,0 +1,47 @@ +services: + datahub-frontend-react: + environment: + - ENABLE_PROMETHEUS=true + - ENABLE_OTEL=true + - OTEL_TRACES_EXPORTER=jaeger + - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250 + - OTEL_METRICS_EXPORTER=none + - OTEL_SERVICE_NAME=datahub-gms + ports: + - "4318" + datahub-gms: + environment: + - ENABLE_PROMETHEUS=true + - ENABLE_OTEL=true + - OTEL_TRACES_EXPORTER=jaeger + - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250 + - OTEL_METRICS_EXPORTER=none + - OTEL_SERVICE_NAME=datahub-gms + ports: + - "4318" + grafana: + depends_on: + - prometheus + image: grafana/grafana:9.1.4 + ports: + - 3001:3000 + volumes: + - grafana-storage:/var/lib/grafana + - ../monitoring/grafana/datasources:/etc/grafana/provisioning/datasources + - ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards + jaeger-all-in-one: + image: jaegertracing/all-in-one:latest + ports: + - 16686:16686 + - "14268" + - "14250" + prometheus: + container_name: prometheus + image: prom/prometheus:latest + ports: + - 9089:9090 + volumes: + - ../monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml +version: "2.3" +volumes: + grafana-storage: null diff --git a/docker/quickstart/temp.quickstart.yml b/docker/quickstart/temp.quickstart.yml new file mode 100644 index 0000000000000..4b3557c8dae09 --- /dev/null +++ b/docker/quickstart/temp.quickstart.yml @@ -0,0 +1,201 @@ +networks: + default: + name: datahub_network +services: + broker: + container_name: broker + depends_on: + - zookeeper + environment: + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m + - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false + hostname: broker + image: confluentinc/cp-kafka:5.4.0 + ports: + - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 + volumes: + - broker:/var/lib/kafka/data/ + datahub-actions: + depends_on: + - datahub-gms + environment: + - DATAHUB_GMS_PROTOCOL=http + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - SCHEMA_REGISTRY_URL=http://schema-registry:8081 + - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 + - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 + - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system + - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT + hostname: actions + image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} + restart: on-failure:5 + datahub-frontend-react: + container_name: datahub-frontend-react + depends_on: + - datahub-gms + environment: + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - DATAHUB_SECRET=YouKnowNothing + - DATAHUB_APP_VERSION=1.0 + - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB + - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 + - ELASTIC_CLIENT_HOST=elasticsearch + - ELASTIC_CLIENT_PORT=9200 + hostname: datahub-frontend-react + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} + ports: + - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 + volumes: + - ${HOME}/.datahub/plugins:/etc/datahub/plugins + datahub-gms: + container_name: datahub-gms + depends_on: + - neo4j + - mysql + environment: + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - EBEAN_DATASOURCE_USERNAME=datahub + - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - NEO4J_HOST=http://neo4j:7474 + - NEO4J_URI=bolt://neo4j + - NEO4J_USERNAME=neo4j + - NEO4J_PASSWORD=datahub + - JAVA_OPTS=-Xms1g -Xmx1g + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - GRAPH_SERVICE_IMPL=neo4j + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - ENTITY_SERVICE_ENABLE_RETENTION=true + - MAE_CONSUMER_ENABLED=true + - MCE_CONSUMER_ENABLED=true + - PE_CONSUMER_ENABLED=true + - UI_INGESTION_ENABLED=true + - METADATA_SERVICE_AUTH_ENABLED=false + hostname: datahub-gms + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} + ports: + - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 + volumes: + - ${HOME}/.datahub/plugins/:/etc/datahub/plugins + - ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources + elasticsearch: + container_name: elasticsearch + environment: + - discovery.type=single-node + - xpack.security.enabled=false + - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true + healthcheck: + retries: 4 + start_period: 2m + test: + - CMD-SHELL + - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1 + hostname: elasticsearch + image: elasticsearch:7.9.3 + mem_limit: 1g + ports: + - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 + volumes: + - esdata:/usr/share/elasticsearch/data + elasticsearch-setup: + container_name: elasticsearch-setup + depends_on: + - elasticsearch + environment: + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_PROTOCOL=http + hostname: elasticsearch-setup + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} + mysql: + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password + container_name: mysql + environment: + - MYSQL_DATABASE=datahub + - MYSQL_USER=datahub + - MYSQL_PASSWORD=datahub + - MYSQL_ROOT_PASSWORD=datahub + hostname: mysql + image: mysql:5.7 + ports: + - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 + volumes: + - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql + - mysqldata:/var/lib/mysql + mysql-setup: + container_name: mysql-setup + depends_on: + - mysql + environment: + - MYSQL_HOST=mysql + - MYSQL_PORT=3306 + - MYSQL_USERNAME=datahub + - MYSQL_PASSWORD=datahub + - DATAHUB_DB_NAME=datahub + hostname: mysql-setup + image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} + neo4j: + container_name: neo4j + environment: + - NEO4J_AUTH=neo4j/datahub + - NEO4J_dbms_default__database=graph.db + - NEO4J_dbms_allow__upgrade=true + hostname: neo4j + image: neo4j:4.4.9-community + ports: + - ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474 + - ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687 + volumes: + - neo4jdata:/data + schema-registry: + container_name: schema-registry + depends_on: + - zookeeper + - broker + environment: + - SCHEMA_REGISTRY_HOST_NAME=schemaregistry + - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 + hostname: schema-registry + image: confluentinc/cp-schema-registry:5.4.0 + ports: + - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 + zookeeper: + container_name: zookeeper + environment: + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 + hostname: zookeeper + image: confluentinc/cp-zookeeper:5.4.0 + ports: + - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 + volumes: + - zkdata:/var/lib/zookeeper +version: "2.3" +volumes: + broker: null + esdata: null + mysqldata: null + neo4jdata: null + zkdata: null diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py index 8dd757b33bda9..d2c3a4b59394a 100644 --- a/metadata-ingestion/src/datahub/cli/docker_cli.py +++ b/metadata-ingestion/src/datahub/cli/docker_cli.py @@ -43,7 +43,10 @@ ELASTIC_QUICKSTART_COMPOSE_FILE = ( "docker/quickstart/docker-compose-without-neo4j.quickstart.yml" ) -M1_QUICKSTART_COMPOSE_FILE = ( +NEO4J_AND_ELASTIC_M1_QUICKSTART_COMPOSE_FILE = ( + "docker/quickstart/docker-compose-m1.quickstart.yml" +) +ELASTIC_M1_QUICKSTART_COMPOSE_FILE = ( "docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml" ) CONSUMERS_QUICKSTART_COMPOSE_FILE = ( @@ -61,8 +64,12 @@ ELASTIC_QUICKSTART_COMPOSE_URL = ( f"{DOCKER_COMPOSE_BASE}/{ELASTIC_QUICKSTART_COMPOSE_FILE}" ) -M1_QUICKSTART_COMPOSE_URL = f"{DOCKER_COMPOSE_BASE}/{M1_QUICKSTART_COMPOSE_FILE}" - +NEO4J_AND_ELASTIC_M1_QUICKSTART_COMPOSE_URL = ( + f"{DOCKER_COMPOSE_BASE}/{NEO4J_AND_ELASTIC_M1_QUICKSTART_COMPOSE_FILE}" +) +ELASTIC_M1_QUICKSTART_COMPOSE_URL = ( + f"{DOCKER_COMPOSE_BASE}/{ELASTIC_M1_QUICKSTART_COMPOSE_FILE}" +) class Architectures(Enum): x86 = "x86" @@ -672,13 +679,13 @@ def quickstart( "Running with neo4j on M1 is not currently supported, will be using elasticsearch as graph", fg="red", ) - github_file = ( - NEO4J_AND_ELASTIC_QUICKSTART_COMPOSE_URL - if should_use_neo4j and not is_arch_m1(quickstart_arch) - else ELASTIC_QUICKSTART_COMPOSE_URL - if not is_arch_m1(quickstart_arch) - else M1_QUICKSTART_COMPOSE_URL - ) + + if should_use_neo4j: + github_file = NEO4J_AND_ELASTIC_QUICKSTART_COMPOSE_URL if not is_arch_m1(quickstart_arch) \ + else NEO4J_AND_ELASTIC_M1_QUICKSTART_COMPOSE_URL + else: + github_file = ELASTIC_QUICKSTART_COMPOSE_URL if not is_arch_m1(quickstart_arch) \ + else ELASTIC_M1_QUICKSTART_COMPOSE_FILE # also allow local files request_session = requests.Session() From bcc1c545ea3d5b888eb02b4aa8e8add2b8ef0c0e Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Thu, 19 Jan 2023 16:15:03 +0000 Subject: [PATCH 10/22] Remove temp files --- docker/quickstart/temp-m1.quickstart.yml | 204 ------------------ .../temp-without-neo4j-m1.quickstart.yml | 180 ---------------- .../temp-without-neo4j.quickstart.yml | 177 --------------- ...emp.consumers-without-neo4j.quickstart.yml | 54 ----- .../quickstart/temp.consumers.quickstart.yml | 66 ------ .../quickstart/temp.monitoring.quickstart.yml | 47 ---- docker/quickstart/temp.quickstart.yml | 201 ----------------- 7 files changed, 929 deletions(-) delete mode 100644 docker/quickstart/temp-m1.quickstart.yml delete mode 100644 docker/quickstart/temp-without-neo4j-m1.quickstart.yml delete mode 100644 docker/quickstart/temp-without-neo4j.quickstart.yml delete mode 100644 docker/quickstart/temp.consumers-without-neo4j.quickstart.yml delete mode 100644 docker/quickstart/temp.consumers.quickstart.yml delete mode 100644 docker/quickstart/temp.monitoring.quickstart.yml delete mode 100644 docker/quickstart/temp.quickstart.yml diff --git a/docker/quickstart/temp-m1.quickstart.yml b/docker/quickstart/temp-m1.quickstart.yml deleted file mode 100644 index d19c68ac526a0..0000000000000 --- a/docker/quickstart/temp-m1.quickstart.yml +++ /dev/null @@ -1,204 +0,0 @@ -networks: - default: - name: datahub_network -services: - broker: - container_name: broker - depends_on: - - zookeeper - environment: - - KAFKA_BROKER_ID=1 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false - hostname: broker - image: kymeric/cp-kafka:latest - ports: - - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 - volumes: - - broker:/var/lib/kafka/data/ - datahub-actions: - depends_on: - - datahub-gms - environment: - - DATAHUB_GMS_PROTOCOL=http - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - SCHEMA_REGISTRY_URL=http://schema-registry:8081 - - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 - - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT - hostname: actions - image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} - restart: on-failure:5 - datahub-frontend-react: - container_name: datahub-frontend-react - depends_on: - - datahub-gms - environment: - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - DATAHUB_SECRET=YouKnowNothing - - DATAHUB_APP_VERSION=1.0 - - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf - -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml - -Dlogback.debug=false -Dpidfile.path=/dev/null - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - - ELASTIC_CLIENT_HOST=elasticsearch - - ELASTIC_CLIENT_PORT=9200 - hostname: datahub-frontend-react - image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} - ports: - - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 - volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins - datahub-gms: - container_name: datahub-gms - depends_on: - - neo4j - - mysql - environment: - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - EBEAN_DATASOURCE_USERNAME=datahub - - EBEAN_DATASOURCE_PASSWORD=datahub - - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - NEO4J_HOST=http://neo4j:7474 - - NEO4J_URI=bolt://neo4j - - NEO4J_USERNAME=neo4j - - NEO4J_PASSWORD=datahub - - JAVA_OPTS=-Xms1g -Xmx1g - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - GRAPH_SERVICE_IMPL=neo4j - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - ENTITY_SERVICE_ENABLE_RETENTION=true - - MAE_CONSUMER_ENABLED=true - - MCE_CONSUMER_ENABLED=true - - PE_CONSUMER_ENABLED=true - - UI_INGESTION_ENABLED=true - - METADATA_SERVICE_AUTH_ENABLED=false - hostname: datahub-gms - image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} - ports: - - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 - volumes: - - ${HOME}/.datahub/plugins/:/etc/datahub/plugins - - ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources - elasticsearch: - container_name: elasticsearch - environment: - - discovery.type=single-node - - xpack.security.enabled=false - - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true - healthcheck: - retries: 4 - start_period: 2m - test: - - CMD-SHELL - - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' - || exit 1 - hostname: elasticsearch - image: elasticsearch:7.9.3 - mem_limit: 1g - ports: - - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 - volumes: - - esdata:/usr/share/elasticsearch/data - elasticsearch-setup: - container_name: elasticsearch-setup - depends_on: - - elasticsearch - environment: - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_PROTOCOL=http - hostname: elasticsearch-setup - image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} - mysql: - command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password - container_name: mysql - environment: - - MYSQL_DATABASE=datahub - - MYSQL_USER=datahub - - MYSQL_PASSWORD=datahub - - MYSQL_ROOT_PASSWORD=datahub - hostname: mysql - image: mariadb:10.5.8 - ports: - - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 - volumes: - - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - - mysqldata:/var/lib/mysql - mysql-setup: - container_name: mysql-setup - depends_on: - - mysql - environment: - - MYSQL_HOST=mysql - - MYSQL_PORT=3306 - - MYSQL_USERNAME=datahub - - MYSQL_PASSWORD=datahub - - DATAHUB_DB_NAME=datahub - hostname: mysql-setup - image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} - neo4j: - container_name: neo4j - environment: - - NEO4J_AUTH=neo4j/datahub - - NEO4J_dbms_default__database=graph.db - - NEO4J_dbms_allow__upgrade=true - hostname: neo4j - image: neo4j/neo4j-arm64-experimental:4.0.6-arm64 - ports: - - ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474 - - ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687 - volumes: - - neo4jdata:/data - schema-registry: - container_name: schema-registry - depends_on: - - zookeeper - - broker - environment: - - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 - hostname: schema-registry - image: eugenetea/schema-registry-arm64:latest - ports: - - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 - zookeeper: - container_name: zookeeper - environment: - - ZOOKEEPER_CLIENT_PORT=2181 - - ZOOKEEPER_TICK_TIME=2000 - hostname: zookeeper - image: kymeric/cp-zookeeper:latest - ports: - - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 - volumes: - - zkdata:/var/lib/zookeeper -version: '2.3' -volumes: - broker: null - esdata: null - mysqldata: null - neo4jdata: null - zkdata: null diff --git a/docker/quickstart/temp-without-neo4j-m1.quickstart.yml b/docker/quickstart/temp-without-neo4j-m1.quickstart.yml deleted file mode 100644 index 06587b31fbe33..0000000000000 --- a/docker/quickstart/temp-without-neo4j-m1.quickstart.yml +++ /dev/null @@ -1,180 +0,0 @@ -networks: - default: - name: datahub_network -services: - broker: - container_name: broker - depends_on: - - zookeeper - environment: - - KAFKA_BROKER_ID=1 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false - hostname: broker - image: kymeric/cp-kafka:latest - ports: - - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 - datahub-actions: - depends_on: - - datahub-gms - environment: - - DATAHUB_GMS_PROTOCOL=http - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - SCHEMA_REGISTRY_URL=http://schema-registry:8081 - - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 - - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT - hostname: actions - image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} - restart: on-failure:5 - datahub-frontend-react: - container_name: datahub-frontend-react - depends_on: - - datahub-gms - environment: - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - DATAHUB_SECRET=YouKnowNothing - - DATAHUB_APP_VERSION=1.0 - - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf - -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml - -Dlogback.debug=false -Dpidfile.path=/dev/null - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - - ELASTIC_CLIENT_HOST=elasticsearch - - ELASTIC_CLIENT_PORT=9200 - hostname: datahub-frontend-react - image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} - ports: - - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 - volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins - datahub-gms: - container_name: datahub-gms - depends_on: - - mysql - environment: - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - JAVA_OPTS=-Xms1g -Xmx1g - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - MCE_CONSUMER_ENABLED=true - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - UI_INGESTION_ENABLED=true - - MAE_CONSUMER_ENABLED=true - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_HOST=elasticsearch - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - EBEAN_DATASOURCE_PASSWORD=datahub - - PE_CONSUMER_ENABLED=true - - ENTITY_SERVICE_ENABLE_RETENTION=true - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - GRAPH_SERVICE_IMPL=elasticsearch - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_USERNAME=datahub - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - hostname: datahub-gms - image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} - ports: - - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 - volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins - elasticsearch: - container_name: elasticsearch - environment: - - discovery.type=single-node - - xpack.security.enabled=false - - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true - healthcheck: - retries: 4 - start_period: 2m - test: - - CMD-SHELL - - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' - || exit 1 - hostname: elasticsearch - image: elasticsearch:7.9.3 - mem_limit: 1g - ports: - - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 - volumes: - - esdata:/usr/share/elasticsearch/data - elasticsearch-setup: - container_name: elasticsearch-setup - depends_on: - - elasticsearch - environment: - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_PROTOCOL=http - hostname: elasticsearch-setup - image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} - mysql: - command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password - container_name: mysql - environment: - - MYSQL_DATABASE=datahub - - MYSQL_USER=datahub - - MYSQL_PASSWORD=datahub - - MYSQL_ROOT_PASSWORD=datahub - hostname: mysql - image: mariadb:10.5.8 - ports: - - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 - volumes: - - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - - mysqldata:/var/lib/mysql - mysql-setup: - container_name: mysql-setup - depends_on: - - mysql - environment: - - MYSQL_HOST=mysql - - MYSQL_PORT=3306 - - MYSQL_USERNAME=datahub - - MYSQL_PASSWORD=datahub - - DATAHUB_DB_NAME=datahub - hostname: mysql-setup - image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} - schema-registry: - container_name: schema-registry - depends_on: - - zookeeper - - broker - environment: - - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 - hostname: schema-registry - image: eugenetea/schema-registry-arm64:latest - ports: - - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 - zookeeper: - container_name: zookeeper - environment: - - ZOOKEEPER_CLIENT_PORT=2181 - - ZOOKEEPER_TICK_TIME=2000 - hostname: zookeeper - image: kymeric/cp-zookeeper:latest - ports: - - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 - volumes: - - zkdata:/var/lib/zookeeper -version: '2.3' -volumes: - esdata: null - mysqldata: null - zkdata: null diff --git a/docker/quickstart/temp-without-neo4j.quickstart.yml b/docker/quickstart/temp-without-neo4j.quickstart.yml deleted file mode 100644 index d388197d8b04f..0000000000000 --- a/docker/quickstart/temp-without-neo4j.quickstart.yml +++ /dev/null @@ -1,177 +0,0 @@ -networks: - default: - name: datahub_network -services: - broker: - container_name: broker - depends_on: - - zookeeper - environment: - - KAFKA_BROKER_ID=1 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false - hostname: broker - image: confluentinc/cp-kafka:5.4.0 - ports: - - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 - datahub-actions: - depends_on: - - datahub-gms - environment: - - DATAHUB_GMS_PROTOCOL=http - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - SCHEMA_REGISTRY_URL=http://schema-registry:8081 - - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 - - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT - hostname: actions - image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} - restart: on-failure:5 - datahub-frontend-react: - container_name: datahub-frontend-react - depends_on: - - datahub-gms - environment: - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - DATAHUB_SECRET=YouKnowNothing - - DATAHUB_APP_VERSION=1.0 - - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - - ELASTIC_CLIENT_HOST=elasticsearch - - ELASTIC_CLIENT_PORT=9200 - hostname: datahub-frontend-react - image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} - ports: - - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 - volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins - datahub-gms: - container_name: datahub-gms - depends_on: - - mysql - environment: - - EBEAN_DATASOURCE_USERNAME=datahub - - ELASTICSEARCH_PORT=9200 - - GRAPH_SERVICE_IMPL=elasticsearch - - UI_INGESTION_ENABLED=true - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - MCE_CONSUMER_ENABLED=true - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - JAVA_OPTS=-Xms1g -Xmx1g - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - EBEAN_DATASOURCE_HOST=mysql:3306 - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - MAE_CONSUMER_ENABLED=true - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - PE_CONSUMER_ENABLED=true - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - ENTITY_SERVICE_ENABLE_RETENTION=true - - EBEAN_DATASOURCE_PASSWORD=datahub - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - hostname: datahub-gms - image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} - ports: - - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 - volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins - elasticsearch: - container_name: elasticsearch - environment: - - discovery.type=single-node - - xpack.security.enabled=false - - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true - healthcheck: - retries: 4 - start_period: 2m - test: - - CMD-SHELL - - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1 - hostname: elasticsearch - image: elasticsearch:7.9.3 - mem_limit: 1g - ports: - - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 - volumes: - - esdata:/usr/share/elasticsearch/data - elasticsearch-setup: - container_name: elasticsearch-setup - depends_on: - - elasticsearch - environment: - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_PROTOCOL=http - hostname: elasticsearch-setup - image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} - mysql: - command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password - container_name: mysql - environment: - - MYSQL_DATABASE=datahub - - MYSQL_USER=datahub - - MYSQL_PASSWORD=datahub - - MYSQL_ROOT_PASSWORD=datahub - hostname: mysql - image: mysql:5.7 - ports: - - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 - volumes: - - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - - mysqldata:/var/lib/mysql - mysql-setup: - container_name: mysql-setup - depends_on: - - mysql - environment: - - MYSQL_HOST=mysql - - MYSQL_PORT=3306 - - MYSQL_USERNAME=datahub - - MYSQL_PASSWORD=datahub - - DATAHUB_DB_NAME=datahub - hostname: mysql-setup - image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} - schema-registry: - container_name: schema-registry - depends_on: - - zookeeper - - broker - environment: - - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 - hostname: schema-registry - image: confluentinc/cp-schema-registry:5.4.0 - ports: - - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 - zookeeper: - container_name: zookeeper - environment: - - ZOOKEEPER_CLIENT_PORT=2181 - - ZOOKEEPER_TICK_TIME=2000 - hostname: zookeeper - image: confluentinc/cp-zookeeper:5.4.0 - ports: - - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 - volumes: - - zkdata:/var/lib/zookeeper -version: "2.3" -volumes: - esdata: null - mysqldata: null - zkdata: null diff --git a/docker/quickstart/temp.consumers-without-neo4j.quickstart.yml b/docker/quickstart/temp.consumers-without-neo4j.quickstart.yml deleted file mode 100644 index cee66989456e7..0000000000000 --- a/docker/quickstart/temp.consumers-without-neo4j.quickstart.yml +++ /dev/null @@ -1,54 +0,0 @@ -services: - datahub-gms: - environment: - - MAE_CONSUMER_ENABLED=false - - MCE_CONSUMER_ENABLED=false - datahub-mae-consumer: - container_name: datahub-mae-consumer - environment: - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - MAE_CONSUMER_ENABLED=true - - PE_CONSUMER_ENABLED=true - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - GRAPH_SERVICE_IMPL=elasticsearch - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml - hostname: datahub-mae-consumer - image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head} - ports: - - 9091:9091 - datahub-mce-consumer: - container_name: datahub-mce-consumer - environment: - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - MCE_CONSUMER_ENABLED=true - - EBEAN_DATASOURCE_USERNAME=datahub - - EBEAN_DATASOURCE_PASSWORD=datahub - - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - GRAPH_SERVICE_IMPL=elasticsearch - - JAVA_OPTS=-Xms1g -Xmx1g - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml - - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - ENTITY_SERVICE_ENABLE_RETENTION=true - - MAE_CONSUMER_ENABLED=false - - PE_CONSUMER_ENABLED=false - - UI_INGESTION_ENABLED=false - hostname: datahub-mce-consumer - image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head} - ports: - - 9090:9090 -version: '2.3' diff --git a/docker/quickstart/temp.consumers.quickstart.yml b/docker/quickstart/temp.consumers.quickstart.yml deleted file mode 100644 index 8dbe776a183e2..0000000000000 --- a/docker/quickstart/temp.consumers.quickstart.yml +++ /dev/null @@ -1,66 +0,0 @@ -services: - datahub-gms: - environment: - - MAE_CONSUMER_ENABLED=false - - MCE_CONSUMER_ENABLED=false - datahub-mae-consumer: - container_name: datahub-mae-consumer - depends_on: - - neo4j - environment: - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - MAE_CONSUMER_ENABLED=true - - PE_CONSUMER_ENABLED=true - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - NEO4J_HOST=http://neo4j:7474 - - NEO4J_URI=bolt://neo4j - - NEO4J_USERNAME=neo4j - - NEO4J_PASSWORD=datahub - - GRAPH_SERVICE_IMPL=neo4j - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml - hostname: datahub-mae-consumer - image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head} - ports: - - 9091:9091 - datahub-mce-consumer: - container_name: datahub-mce-consumer - depends_on: - - neo4j - environment: - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - NEO4J_HOST=http://neo4j:7474 - - NEO4J_URI=bolt://neo4j - - NEO4J_USERNAME=neo4j - - NEO4J_PASSWORD=datahub - - GRAPH_SERVICE_IMPL=neo4j - - MCE_CONSUMER_ENABLED=true - - EBEAN_DATASOURCE_USERNAME=datahub - - EBEAN_DATASOURCE_PASSWORD=datahub - - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - JAVA_OPTS=-Xms1g -Xmx1g - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml - - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - ENTITY_SERVICE_ENABLE_RETENTION=true - - MAE_CONSUMER_ENABLED=false - - PE_CONSUMER_ENABLED=false - - UI_INGESTION_ENABLED=false - hostname: datahub-mce-consumer - image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head} - ports: - - 9090:9090 -version: '2.3' diff --git a/docker/quickstart/temp.monitoring.quickstart.yml b/docker/quickstart/temp.monitoring.quickstart.yml deleted file mode 100644 index 37ccd057a00ed..0000000000000 --- a/docker/quickstart/temp.monitoring.quickstart.yml +++ /dev/null @@ -1,47 +0,0 @@ -services: - datahub-frontend-react: - environment: - - ENABLE_PROMETHEUS=true - - ENABLE_OTEL=true - - OTEL_TRACES_EXPORTER=jaeger - - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250 - - OTEL_METRICS_EXPORTER=none - - OTEL_SERVICE_NAME=datahub-gms - ports: - - "4318" - datahub-gms: - environment: - - ENABLE_PROMETHEUS=true - - ENABLE_OTEL=true - - OTEL_TRACES_EXPORTER=jaeger - - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250 - - OTEL_METRICS_EXPORTER=none - - OTEL_SERVICE_NAME=datahub-gms - ports: - - "4318" - grafana: - depends_on: - - prometheus - image: grafana/grafana:9.1.4 - ports: - - 3001:3000 - volumes: - - grafana-storage:/var/lib/grafana - - ../monitoring/grafana/datasources:/etc/grafana/provisioning/datasources - - ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards - jaeger-all-in-one: - image: jaegertracing/all-in-one:latest - ports: - - 16686:16686 - - "14268" - - "14250" - prometheus: - container_name: prometheus - image: prom/prometheus:latest - ports: - - 9089:9090 - volumes: - - ../monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml -version: "2.3" -volumes: - grafana-storage: null diff --git a/docker/quickstart/temp.quickstart.yml b/docker/quickstart/temp.quickstart.yml deleted file mode 100644 index 4b3557c8dae09..0000000000000 --- a/docker/quickstart/temp.quickstart.yml +++ /dev/null @@ -1,201 +0,0 @@ -networks: - default: - name: datahub_network -services: - broker: - container_name: broker - depends_on: - - zookeeper - environment: - - KAFKA_BROKER_ID=1 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false - hostname: broker - image: confluentinc/cp-kafka:5.4.0 - ports: - - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 - volumes: - - broker:/var/lib/kafka/data/ - datahub-actions: - depends_on: - - datahub-gms - environment: - - DATAHUB_GMS_PROTOCOL=http - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - SCHEMA_REGISTRY_URL=http://schema-registry:8081 - - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 - - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT - hostname: actions - image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} - restart: on-failure:5 - datahub-frontend-react: - container_name: datahub-frontend-react - depends_on: - - datahub-gms - environment: - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - DATAHUB_SECRET=YouKnowNothing - - DATAHUB_APP_VERSION=1.0 - - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - - ELASTIC_CLIENT_HOST=elasticsearch - - ELASTIC_CLIENT_PORT=9200 - hostname: datahub-frontend-react - image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} - ports: - - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 - volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins - datahub-gms: - container_name: datahub-gms - depends_on: - - neo4j - - mysql - environment: - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - EBEAN_DATASOURCE_USERNAME=datahub - - EBEAN_DATASOURCE_PASSWORD=datahub - - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - NEO4J_HOST=http://neo4j:7474 - - NEO4J_URI=bolt://neo4j - - NEO4J_USERNAME=neo4j - - NEO4J_PASSWORD=datahub - - JAVA_OPTS=-Xms1g -Xmx1g - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - GRAPH_SERVICE_IMPL=neo4j - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - ENTITY_SERVICE_ENABLE_RETENTION=true - - MAE_CONSUMER_ENABLED=true - - MCE_CONSUMER_ENABLED=true - - PE_CONSUMER_ENABLED=true - - UI_INGESTION_ENABLED=true - - METADATA_SERVICE_AUTH_ENABLED=false - hostname: datahub-gms - image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} - ports: - - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 - volumes: - - ${HOME}/.datahub/plugins/:/etc/datahub/plugins - - ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources - elasticsearch: - container_name: elasticsearch - environment: - - discovery.type=single-node - - xpack.security.enabled=false - - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true - healthcheck: - retries: 4 - start_period: 2m - test: - - CMD-SHELL - - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1 - hostname: elasticsearch - image: elasticsearch:7.9.3 - mem_limit: 1g - ports: - - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 - volumes: - - esdata:/usr/share/elasticsearch/data - elasticsearch-setup: - container_name: elasticsearch-setup - depends_on: - - elasticsearch - environment: - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_PROTOCOL=http - hostname: elasticsearch-setup - image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} - mysql: - command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password - container_name: mysql - environment: - - MYSQL_DATABASE=datahub - - MYSQL_USER=datahub - - MYSQL_PASSWORD=datahub - - MYSQL_ROOT_PASSWORD=datahub - hostname: mysql - image: mysql:5.7 - ports: - - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 - volumes: - - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - - mysqldata:/var/lib/mysql - mysql-setup: - container_name: mysql-setup - depends_on: - - mysql - environment: - - MYSQL_HOST=mysql - - MYSQL_PORT=3306 - - MYSQL_USERNAME=datahub - - MYSQL_PASSWORD=datahub - - DATAHUB_DB_NAME=datahub - hostname: mysql-setup - image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} - neo4j: - container_name: neo4j - environment: - - NEO4J_AUTH=neo4j/datahub - - NEO4J_dbms_default__database=graph.db - - NEO4J_dbms_allow__upgrade=true - hostname: neo4j - image: neo4j:4.4.9-community - ports: - - ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474 - - ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687 - volumes: - - neo4jdata:/data - schema-registry: - container_name: schema-registry - depends_on: - - zookeeper - - broker - environment: - - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 - hostname: schema-registry - image: confluentinc/cp-schema-registry:5.4.0 - ports: - - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 - zookeeper: - container_name: zookeeper - environment: - - ZOOKEEPER_CLIENT_PORT=2181 - - ZOOKEEPER_TICK_TIME=2000 - hostname: zookeeper - image: confluentinc/cp-zookeeper:5.4.0 - ports: - - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 - volumes: - - zkdata:/var/lib/zookeeper -version: "2.3" -volumes: - broker: null - esdata: null - mysqldata: null - neo4jdata: null - zkdata: null From b3eba2207f8e467e2e88f739129c0ad616704cc9 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Thu, 19 Jan 2023 16:27:10 +0000 Subject: [PATCH 11/22] Fix docker cli linting issues --- .../src/datahub/cli/docker_cli.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py index d2c3a4b59394a..e6f72e70b89f9 100644 --- a/metadata-ingestion/src/datahub/cli/docker_cli.py +++ b/metadata-ingestion/src/datahub/cli/docker_cli.py @@ -71,6 +71,7 @@ f"{DOCKER_COMPOSE_BASE}/{ELASTIC_M1_QUICKSTART_COMPOSE_FILE}" ) + class Architectures(Enum): x86 = "x86" arm64 = "arm64" @@ -681,11 +682,17 @@ def quickstart( ) if should_use_neo4j: - github_file = NEO4J_AND_ELASTIC_QUICKSTART_COMPOSE_URL if not is_arch_m1(quickstart_arch) \ + github_file = ( + NEO4J_AND_ELASTIC_QUICKSTART_COMPOSE_URL + if not is_arch_m1(quickstart_arch) else NEO4J_AND_ELASTIC_M1_QUICKSTART_COMPOSE_URL + ) else: - github_file = ELASTIC_QUICKSTART_COMPOSE_URL if not is_arch_m1(quickstart_arch) \ + github_file = ( + ELASTIC_QUICKSTART_COMPOSE_URL + if not is_arch_m1(quickstart_arch) else ELASTIC_M1_QUICKSTART_COMPOSE_FILE + ) # also allow local files request_session = requests.Session() @@ -737,10 +744,10 @@ def quickstart( ) default_consumer_compose_file = ( - Path(DATAHUB_ROOT_FOLDER) / "quickstart/docker-compose.consumers.yml" + Path(DATAHUB_ROOT_FOLDER) / "quickstart/docker-compose.consumers.yml" ) with open( - default_consumer_compose_file, "wb" + default_consumer_compose_file, "wb" ) if default_consumer_compose_file else tempfile.NamedTemporaryFile( suffix=".yml", delete=False ) as tmp_file: @@ -750,7 +757,9 @@ def quickstart( f"Fetching consumer docker-compose file {kafka_setup_github_file} from GitHub" ) # Download the quickstart docker-compose file from GitHub. - quickstart_download_response = request_session.get(kafka_setup_github_file) + quickstart_download_response = request_session.get( + kafka_setup_github_file + ) quickstart_download_response.raise_for_status() tmp_file.write(quickstart_download_response.content) logger.debug(f"Copied to {path}") From 19d3b26dcb89dc64d83624ad2bc1cb9384e91826 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Thu, 19 Jan 2023 16:37:11 +0000 Subject: [PATCH 12/22] Correct url --- metadata-ingestion/src/datahub/cli/docker_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py index e6f72e70b89f9..45d257f7b1d09 100644 --- a/metadata-ingestion/src/datahub/cli/docker_cli.py +++ b/metadata-ingestion/src/datahub/cli/docker_cli.py @@ -691,7 +691,7 @@ def quickstart( github_file = ( ELASTIC_QUICKSTART_COMPOSE_URL if not is_arch_m1(quickstart_arch) - else ELASTIC_M1_QUICKSTART_COMPOSE_FILE + else ELASTIC_M1_QUICKSTART_COMPOSE_URL ) # also allow local files From a905b219c47d709711615bba77bff1bc0d7017e3 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Thu, 19 Jan 2023 16:45:00 +0000 Subject: [PATCH 13/22] remove old code --- metadata-ingestion/src/datahub/cli/docker_cli.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py index 45d257f7b1d09..61d4ed05f3537 100644 --- a/metadata-ingestion/src/datahub/cli/docker_cli.py +++ b/metadata-ingestion/src/datahub/cli/docker_cli.py @@ -176,7 +176,7 @@ def should_use_neo4j_for_graph_service(graph_service_override: Optional[str]) -> click.echo( "No Datahub Neo4j volume found, starting with elasticsearch as graph service.\n" "To use neo4j as a graph backend, run \n" - "`datahub docker quickstart --quickstart-compose-file ./docker/quickstart/docker-compose.quickstart.yml`" + "`datahub docker quickstart --graph-service-impl neo4j`" "\nfrom the root of the datahub repo\n" ) return False @@ -675,11 +675,6 @@ def quickstart( elif not quickstart_compose_file: # download appropriate quickstart file should_use_neo4j = should_use_neo4j_for_graph_service(graph_service_impl) - if should_use_neo4j and is_arch_m1(quickstart_arch): - click.secho( - "Running with neo4j on M1 is not currently supported, will be using elasticsearch as graph", - fg="red", - ) if should_use_neo4j: github_file = ( From a8118ff6eed395fea53e90ae33424ae20beeaf7f Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Thu, 19 Jan 2023 17:24:13 +0000 Subject: [PATCH 14/22] Fix cyclomatic complexity of docker method --- .../src/datahub/cli/docker_cli.py | 185 ++++++++++-------- 1 file changed, 98 insertions(+), 87 deletions(-) diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py index 61d4ed05f3537..475854aae6b2f 100644 --- a/metadata-ingestion/src/datahub/cli/docker_cli.py +++ b/metadata-ingestion/src/datahub/cli/docker_cli.py @@ -661,103 +661,26 @@ def quickstart( if issues: _print_issue_list_and_exit(issues, "Unable to run quickstart:") - quickstart_compose_file = list( + quickstart_compose_file_list = list( quickstart_compose_file ) # convert to list from tuple auth_resources_folder = Path(DATAHUB_ROOT_FOLDER) / "plugins/auth/resources" os.makedirs(auth_resources_folder, exist_ok=True) - default_quickstart_compose_file = _get_default_quickstart_compose_file() + quickstart_compose_file_name = _get_default_quickstart_compose_file() if stop: _attempt_stop(quickstart_compose_file) return elif not quickstart_compose_file: - # download appropriate quickstart file - should_use_neo4j = should_use_neo4j_for_graph_service(graph_service_impl) - - if should_use_neo4j: - github_file = ( - NEO4J_AND_ELASTIC_QUICKSTART_COMPOSE_URL - if not is_arch_m1(quickstart_arch) - else NEO4J_AND_ELASTIC_M1_QUICKSTART_COMPOSE_URL - ) - else: - github_file = ( - ELASTIC_QUICKSTART_COMPOSE_URL - if not is_arch_m1(quickstart_arch) - else ELASTIC_M1_QUICKSTART_COMPOSE_URL - ) - - # also allow local files - request_session = requests.Session() - request_session.mount("file://", FileAdapter()) - - with open( - default_quickstart_compose_file, "wb" - ) if default_quickstart_compose_file else tempfile.NamedTemporaryFile( - suffix=".yml", delete=False - ) as tmp_file: - path = pathlib.Path(tmp_file.name) - quickstart_compose_file.append(path) - click.echo(f"Fetching docker-compose file {github_file} from GitHub") - # Download the quickstart docker-compose file from GitHub. - quickstart_download_response = request_session.get(github_file) - quickstart_download_response.raise_for_status() - tmp_file.write(quickstart_download_response.content) - logger.debug(f"Copied to {path}") - - if standalone_consumers: - consumer_github_file = ( - f"{DOCKER_COMPOSE_BASE}/{CONSUMERS_QUICKSTART_COMPOSE_FILE}" - if should_use_neo4j - else f"{DOCKER_COMPOSE_BASE}/{ELASTIC_CONSUMERS_QUICKSTART_COMPOSE_FILE}" - ) - - default_consumer_compose_file = ( - Path(DATAHUB_ROOT_FOLDER) / "quickstart/docker-compose.consumers.yml" - ) - with open( - default_consumer_compose_file, "wb" - ) if default_consumer_compose_file else tempfile.NamedTemporaryFile( - suffix=".yml", delete=False - ) as tmp_file: - path = pathlib.Path(tmp_file.name) - quickstart_compose_file.append(path) - click.echo( - f"Fetching consumer docker-compose file {consumer_github_file} from GitHub" - ) - # Download the quickstart docker-compose file from GitHub. - quickstart_download_response = request_session.get(consumer_github_file) - quickstart_download_response.raise_for_status() - tmp_file.write(quickstart_download_response.content) - logger.debug(f"Copied to {path}") - - if kafka_setup: - kafka_setup_github_file = ( - f"{DOCKER_COMPOSE_BASE}/{KAFKA_SETUP_QUICKSTART_COMPOSE_FILE}" - ) - - default_consumer_compose_file = ( - Path(DATAHUB_ROOT_FOLDER) / "quickstart/docker-compose.consumers.yml" - ) - with open( - default_consumer_compose_file, "wb" - ) if default_consumer_compose_file else tempfile.NamedTemporaryFile( - suffix=".yml", delete=False - ) as tmp_file: - path = pathlib.Path(tmp_file.name) - quickstart_compose_file.append(path) - click.echo( - f"Fetching consumer docker-compose file {kafka_setup_github_file} from GitHub" - ) - # Download the quickstart docker-compose file from GitHub. - quickstart_download_response = request_session.get( - kafka_setup_github_file - ) - quickstart_download_response.raise_for_status() - tmp_file.write(quickstart_download_response.content) - logger.debug(f"Copied to {path}") + download_compose_files( + quickstart_compose_file_name, + quickstart_compose_file_list, + graph_service_impl, + kafka_setup, + quickstart_arch, + standalone_consumers, + ) # set version _set_environment_variables( @@ -878,6 +801,94 @@ def quickstart( ) +def download_compose_files( + quickstart_compose_file_name, + quickstart_compose_file_list, + graph_service_impl, + kafka_setup, + quickstart_arch, + standalone_consumers, +): + # download appropriate quickstart file + should_use_neo4j = should_use_neo4j_for_graph_service(graph_service_impl) + if should_use_neo4j: + github_file = ( + NEO4J_AND_ELASTIC_QUICKSTART_COMPOSE_URL + if not is_arch_m1(quickstart_arch) + else NEO4J_AND_ELASTIC_M1_QUICKSTART_COMPOSE_URL + ) + else: + github_file = ( + ELASTIC_QUICKSTART_COMPOSE_URL + if not is_arch_m1(quickstart_arch) + else ELASTIC_M1_QUICKSTART_COMPOSE_URL + ) + # also allow local files + request_session = requests.Session() + request_session.mount("file://", FileAdapter()) + with open( + quickstart_compose_file_name, "wb" + ) if quickstart_compose_file_name else tempfile.NamedTemporaryFile( + suffix=".yml", delete=False + ) as tmp_file: + path = pathlib.Path(tmp_file.name) + quickstart_compose_file_list.append(path) + click.echo(f"Fetching docker-compose file {github_file} from GitHub") + # Download the quickstart docker-compose file from GitHub. + quickstart_download_response = request_session.get(github_file) + quickstart_download_response.raise_for_status() + tmp_file.write(quickstart_download_response.content) + logger.debug(f"Copied to {path}") + if standalone_consumers: + consumer_github_file = ( + f"{DOCKER_COMPOSE_BASE}/{CONSUMERS_QUICKSTART_COMPOSE_FILE}" + if should_use_neo4j + else f"{DOCKER_COMPOSE_BASE}/{ELASTIC_CONSUMERS_QUICKSTART_COMPOSE_FILE}" + ) + + default_consumer_compose_file = ( + Path(DATAHUB_ROOT_FOLDER) / "quickstart/docker-compose.consumers.yml" + ) + with open( + default_consumer_compose_file, "wb" + ) if default_consumer_compose_file else tempfile.NamedTemporaryFile( + suffix=".yml", delete=False + ) as tmp_file: + path = pathlib.Path(tmp_file.name) + quickstart_compose_file_list.append(path) + click.echo( + f"Fetching consumer docker-compose file {consumer_github_file} from GitHub" + ) + # Download the quickstart docker-compose file from GitHub. + quickstart_download_response = request_session.get(consumer_github_file) + quickstart_download_response.raise_for_status() + tmp_file.write(quickstart_download_response.content) + logger.debug(f"Copied to {path}") + if kafka_setup: + kafka_setup_github_file = ( + f"{DOCKER_COMPOSE_BASE}/{KAFKA_SETUP_QUICKSTART_COMPOSE_FILE}" + ) + + default_consumer_compose_file = ( + Path(DATAHUB_ROOT_FOLDER) / "quickstart/docker-compose.consumers.yml" + ) + with open( + default_consumer_compose_file, "wb" + ) if default_consumer_compose_file else tempfile.NamedTemporaryFile( + suffix=".yml", delete=False + ) as tmp_file: + path = pathlib.Path(tmp_file.name) + quickstart_compose_file_list.append(path) + click.echo( + f"Fetching consumer docker-compose file {kafka_setup_github_file} from GitHub" + ) + # Download the quickstart docker-compose file from GitHub. + quickstart_download_response = request_session.get(kafka_setup_github_file) + quickstart_download_response.raise_for_status() + tmp_file.write(quickstart_download_response.content) + logger.debug(f"Copied to {path}") + + def valid_restore_options( restore: bool, restore_indices: bool, no_restore_indices: bool ) -> bool: From 4dc63324bc906cf2e0cd850fdb755237634cc658 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Fri, 20 Jan 2023 11:21:21 +0000 Subject: [PATCH 15/22] Use YQ installation Github Action --- .github/workflows/build-and-test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 2f30a4ab583ae..f81f6cebc376b 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -73,9 +73,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.7" - - name: Install yq for yaml parsing - run: VERSION=v4.30.8 and BINARY=yq_linux_amd64 wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY}.tar.gz -O - |\ - tar xz && mv ${BINARY} /usr/bin/yq + - uses: chrisdickinson/setup-yq@4.30.8 - name: Quickstart Compose Validation run: ./docker/quickstart/generate_and_compare.sh From 4af8370dffa93b9940019dcfd3064875aa6b5284 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Fri, 20 Jan 2023 11:36:28 +0000 Subject: [PATCH 16/22] Specify yq version to install --- .github/workflows/build-and-test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index f81f6cebc376b..28f638906d3eb 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -73,7 +73,10 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.7" - - uses: chrisdickinson/setup-yq@4.30.8 + - name: Download YQ + uses: chrisdickinson/setup-yq@1.0.0 + with: + yq-version: 4.28.2 - name: Quickstart Compose Validation run: ./docker/quickstart/generate_and_compare.sh From 7f85331074cbc6ee223442ef1b1f02580f8baa79 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Fri, 20 Jan 2023 11:39:54 +0000 Subject: [PATCH 17/22] Fix action version --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 28f638906d3eb..76b5a94f77b27 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -74,7 +74,7 @@ jobs: with: python-version: "3.7" - name: Download YQ - uses: chrisdickinson/setup-yq@1.0.0 + uses: chrisdickinson/setup-yq@v1.0.0 with: yq-version: 4.28.2 - name: Quickstart Compose Validation From 3b15857b117047b0523312ddef179ce780a5933e Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Fri, 20 Jan 2023 11:44:31 +0000 Subject: [PATCH 18/22] bump to latest action version --- .github/workflows/build-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 76b5a94f77b27..d83dd2766770b 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -74,9 +74,9 @@ jobs: with: python-version: "3.7" - name: Download YQ - uses: chrisdickinson/setup-yq@v1.0.0 + uses: chrisdickinson/setup-yq@v1.0.1 with: - yq-version: 4.28.2 + yq-version: v4.28.2 - name: Quickstart Compose Validation run: ./docker/quickstart/generate_and_compare.sh From 566760df16779a5c2a16fa4fc14bb660aa2bf700 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Fri, 20 Jan 2023 15:48:42 +0000 Subject: [PATCH 19/22] Fix bug in docker cli python file --- metadata-ingestion/src/datahub/cli/docker_cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py index 475854aae6b2f..c1437ee600f36 100644 --- a/metadata-ingestion/src/datahub/cli/docker_cli.py +++ b/metadata-ingestion/src/datahub/cli/docker_cli.py @@ -661,7 +661,7 @@ def quickstart( if issues: _print_issue_list_and_exit(issues, "Unable to run quickstart:") - quickstart_compose_file_list = list( + quickstart_compose_file = list( quickstart_compose_file ) # convert to list from tuple @@ -673,9 +673,10 @@ def quickstart( _attempt_stop(quickstart_compose_file) return elif not quickstart_compose_file: + print("compose file name", quickstart_compose_file_name) download_compose_files( quickstart_compose_file_name, - quickstart_compose_file_list, + quickstart_compose_file, graph_service_impl, kafka_setup, quickstart_arch, From 3003b52eee8dc80a295f8921fb01a2c6e6fcdde5 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Mon, 23 Jan 2023 16:54:48 +0000 Subject: [PATCH 20/22] Update docker images to use kafka 7.2.2 --- docker/docker-compose-with-cassandra.yml | 6 +-- docker/docker-compose-without-neo4j.yml | 6 +-- docker/docker-compose.m1.yml | 6 --- docker/docker-compose.tools.yml | 2 +- docker/docker-compose.yml | 6 +-- .../docker-compose-m1.quickstart.yml | 6 +-- ...er-compose-without-neo4j-m1.quickstart.yml | 32 +++++++-------- ...ocker-compose-without-neo4j.quickstart.yml | 40 +++++++++---------- .../quickstart/docker-compose.quickstart.yml | 6 +-- 9 files changed, 52 insertions(+), 58 deletions(-) diff --git a/docker/docker-compose-with-cassandra.yml b/docker/docker-compose-with-cassandra.yml index c9d316c079c69..24dd3436613fe 100644 --- a/docker/docker-compose-with-cassandra.yml +++ b/docker/docker-compose-with-cassandra.yml @@ -7,7 +7,7 @@ version: '3.8' services: zookeeper: - image: confluentinc/cp-zookeeper:5.4.0 + image: confluentinc/cp-zookeeper:7.2.2 env_file: zookeeper/env/docker.env hostname: zookeeper container_name: zookeeper @@ -17,7 +17,7 @@ services: - zkdata:/var/lib/zookeeper broker: - image: confluentinc/cp-kafka:5.4.0 + image: confluentinc/cp-kafka:7.2.2 env_file: broker/env/docker.env hostname: broker container_name: broker @@ -30,7 +30,7 @@ services: - broker:/var/lib/kafka/data/ schema-registry: - image: confluentinc/cp-schema-registry:5.4.0 + image: confluentinc/cp-schema-registry:7.2.2 env_file: schema-registry/env/docker.env hostname: schema-registry container_name: schema-registry diff --git a/docker/docker-compose-without-neo4j.yml b/docker/docker-compose-without-neo4j.yml index 6e617ffeeef80..2c6b57b564238 100644 --- a/docker/docker-compose-without-neo4j.yml +++ b/docker/docker-compose-without-neo4j.yml @@ -7,7 +7,7 @@ version: '3.8' services: zookeeper: - image: confluentinc/cp-zookeeper:5.4.0 + image: confluentinc/cp-zookeeper:7.2.2 env_file: zookeeper/env/docker.env hostname: zookeeper container_name: zookeeper @@ -17,7 +17,7 @@ services: - zkdata:/var/lib/zookeeper broker: - image: confluentinc/cp-kafka:5.4.0 + image: confluentinc/cp-kafka:7.2.2 env_file: broker/env/docker.env hostname: broker container_name: broker @@ -27,7 +27,7 @@ services: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 schema-registry: - image: confluentinc/cp-schema-registry:5.4.0 + image: confluentinc/cp-schema-registry:7.2.2 env_file: schema-registry/env/docker.env hostname: schema-registry container_name: schema-registry diff --git a/docker/docker-compose.m1.yml b/docker/docker-compose.m1.yml index ea844e852bcf2..368cf795e6ad4 100644 --- a/docker/docker-compose.m1.yml +++ b/docker/docker-compose.m1.yml @@ -1,11 +1,5 @@ services: - broker: - image: kymeric/cp-kafka:latest mysql: image: mariadb:10.5.8 - schema-registry: - image: eugenetea/schema-registry-arm64:latest - zookeeper: - image: kymeric/cp-zookeeper:latest neo4j: image: neo4j/neo4j-arm64-experimental:4.0.6-arm64 diff --git a/docker/docker-compose.tools.yml b/docker/docker-compose.tools.yml index bbb77084bff2e..bc51a226ecd94 100644 --- a/docker/docker-compose.tools.yml +++ b/docker/docker-compose.tools.yml @@ -3,7 +3,7 @@ version: '3.8' services: kafka-rest-proxy: - image: confluentinc/cp-kafka-rest:5.4.0 + image: confluentinc/cp-kafka-rest:7.2.2 env_file: kafka-rest-proxy/env/docker.env hostname: kafka-rest-proxy container_name: kafka-rest-proxy diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a1eff1e752ded..b193029622f90 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -7,7 +7,7 @@ version: '3.8' services: zookeeper: - image: confluentinc/cp-zookeeper:5.4.0 + image: confluentinc/cp-zookeeper:7.2.2 env_file: zookeeper/env/docker.env hostname: zookeeper container_name: zookeeper @@ -17,7 +17,7 @@ services: - zkdata:/var/lib/zookeeper broker: - image: confluentinc/cp-kafka:5.4.0 + image: confluentinc/cp-kafka:7.2.2 env_file: broker/env/docker.env hostname: broker container_name: broker @@ -29,7 +29,7 @@ services: - broker:/var/lib/kafka/data/ schema-registry: - image: confluentinc/cp-schema-registry:5.4.0 + image: confluentinc/cp-schema-registry:7.2.2 env_file: schema-registry/env/docker.env hostname: schema-registry container_name: schema-registry diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml index d19c68ac526a0..d1b0e00850189 100644 --- a/docker/quickstart/docker-compose-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-m1.quickstart.yml @@ -16,7 +16,7 @@ services: - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false hostname: broker - image: kymeric/cp-kafka:latest + image: confluentinc/cp-kafka:7.2.2 ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: @@ -181,7 +181,7 @@ services: - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 hostname: schema-registry - image: eugenetea/schema-registry-arm64:latest + image: confluentinc/cp-schema-registry:7.2.2 ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: @@ -190,7 +190,7 @@ services: - ZOOKEEPER_CLIENT_PORT=2181 - ZOOKEEPER_TICK_TIME=2000 hostname: zookeeper - image: kymeric/cp-zookeeper:latest + image: confluentinc/cp-zookeeper:7.2.2 ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index 8359393381aec..f4dcb00a72663 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -64,29 +64,29 @@ services: depends_on: - mysql environment: - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - UI_INGESTION_ENABLED=true - GRAPH_SERVICE_IMPL=elasticsearch - EBEAN_DATASOURCE_PASSWORD=datahub - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - EBEAN_DATASOURCE_USERNAME=datahub - - MAE_CONSUMER_ENABLED=true - - JAVA_OPTS=-Xms1g -Xmx1g + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - ELASTICSEARCH_PORT=9200 + - EBEAN_DATASOURCE_HOST=mysql:3306 - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_PORT=9200 + - JAVA_OPTS=-Xms1g -Xmx1g + - PE_CONSUMER_ENABLED=true + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - MCE_CONSUMER_ENABLED=true - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - EBEAN_DATASOURCE_HOST=mysql:3306 - ENTITY_SERVICE_ENABLE_RETENTION=true - - ELASTICSEARCH_HOST=elasticsearch - - UI_INGESTION_ENABLED=true - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - EBEAN_DATASOURCE_USERNAME=datahub + - MAE_CONSUMER_ENABLED=true - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - PE_CONSUMER_ENABLED=true + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 064f21a32c59e..759caf5e1d0d2 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -16,7 +16,7 @@ services: - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false hostname: broker - image: confluentinc/cp-kafka:5.4.0 + image: confluentinc/cp-kafka:7.2.2 ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 datahub-actions: @@ -62,29 +62,29 @@ services: depends_on: - mysql environment: - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ELASTICSEARCH_PORT=9200 - MAE_CONSUMER_ENABLED=true - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - EBEAN_DATASOURCE_USERNAME=datahub - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - MCE_CONSUMER_ENABLED=true - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - JAVA_OPTS=-Xms1g -Xmx1g - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - PE_CONSUMER_ENABLED=true - - EBEAN_DATASOURCE_HOST=mysql:3306 - - GRAPH_SERVICE_IMPL=elasticsearch - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - EBEAN_DATASOURCE_PASSWORD=datahub - - ENTITY_SERVICE_ENABLE_RETENTION=true - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - ELASTICSEARCH_HOST=elasticsearch - - UI_INGESTION_ENABLED=true + - ENTITY_SERVICE_ENABLE_RETENTION=true - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - EBEAN_DATASOURCE_USERNAME=datahub + - EBEAN_DATASOURCE_HOST=mysql:3306 - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - EBEAN_DATASOURCE_PASSWORD=datahub + - UI_INGESTION_ENABLED=true + - JAVA_OPTS=-Xms1g -Xmx1g + - GRAPH_SERVICE_IMPL=elasticsearch + - MCE_CONSUMER_ENABLED=true + - PE_CONSUMER_ENABLED=true + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: @@ -156,7 +156,7 @@ services: - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 hostname: schema-registry - image: confluentinc/cp-schema-registry:5.4.0 + image: confluentinc/cp-schema-registry:7.2.2 ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: @@ -165,7 +165,7 @@ services: - ZOOKEEPER_CLIENT_PORT=2181 - ZOOKEEPER_TICK_TIME=2000 hostname: zookeeper - image: confluentinc/cp-zookeeper:5.4.0 + image: confluentinc/cp-zookeeper:7.2.2 ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 4b3557c8dae09..aae1bebfac681 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -16,7 +16,7 @@ services: - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false hostname: broker - image: confluentinc/cp-kafka:5.4.0 + image: confluentinc/cp-kafka:7.2.2 ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: @@ -178,7 +178,7 @@ services: - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 hostname: schema-registry - image: confluentinc/cp-schema-registry:5.4.0 + image: confluentinc/cp-schema-registry:7.2.2 ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: @@ -187,7 +187,7 @@ services: - ZOOKEEPER_CLIENT_PORT=2181 - ZOOKEEPER_TICK_TIME=2000 hostname: zookeeper - image: confluentinc/cp-zookeeper:5.4.0 + image: confluentinc/cp-zookeeper:7.2.2 ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: From c8a46029d884903b5e15ac13e688931a128247f6 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Mon, 23 Jan 2023 18:24:54 +0000 Subject: [PATCH 21/22] Remove kymeric reference --- docker/docker-compose-without-neo4j.m1.yml | 6 --- ...er-compose-without-neo4j-m1.quickstart.yml | 40 +++++++++---------- ...ocker-compose-without-neo4j.quickstart.yml | 30 +++++++------- 3 files changed, 35 insertions(+), 41 deletions(-) diff --git a/docker/docker-compose-without-neo4j.m1.yml b/docker/docker-compose-without-neo4j.m1.yml index da47c7944045b..f8f98ce13d9b8 100644 --- a/docker/docker-compose-without-neo4j.m1.yml +++ b/docker/docker-compose-without-neo4j.m1.yml @@ -1,9 +1,3 @@ services: - broker: - image: kymeric/cp-kafka:latest mysql: image: mariadb:10.5.8 - schema-registry: - image: eugenetea/schema-registry-arm64:latest - zookeeper: - image: kymeric/cp-zookeeper:latest diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index f4dcb00a72663..92c0c51c9fa21 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -16,7 +16,7 @@ services: - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false hostname: broker - image: kymeric/cp-kafka:latest + image: confluentinc/cp-kafka:7.2.2 ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 datahub-actions: @@ -64,29 +64,29 @@ services: depends_on: - mysql environment: - - UI_INGESTION_ENABLED=true + - EBEAN_DATASOURCE_HOST=mysql:3306 + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - GRAPH_SERVICE_IMPL=elasticsearch - - EBEAN_DATASOURCE_PASSWORD=datahub + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - MAE_CONSUMER_ENABLED=true + - UI_INGESTION_ENABLED=true + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - PE_CONSUMER_ENABLED=true + - JAVA_OPTS=-Xms1g -Xmx1g + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - ELASTICSEARCH_HOST=elasticsearch + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - ELASTICSEARCH_PORT=9200 - - EBEAN_DATASOURCE_HOST=mysql:3306 - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - EBEAN_DATASOURCE_USERNAME=datahub + - MCE_CONSUMER_ENABLED=true - KAFKA_BOOTSTRAP_SERVER=broker:29092 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - JAVA_OPTS=-Xms1g -Xmx1g - - PE_CONSUMER_ENABLED=true - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - MCE_CONSUMER_ENABLED=true + - EBEAN_DATASOURCE_PASSWORD=datahub - ENTITY_SERVICE_ENABLE_RETENTION=true - - EBEAN_DATASOURCE_USERNAME=datahub - - MAE_CONSUMER_ENABLED=true - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: @@ -159,7 +159,7 @@ services: - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 hostname: schema-registry - image: eugenetea/schema-registry-arm64:latest + image: confluentinc/cp-schema-registry:7.2.2 ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: @@ -168,7 +168,7 @@ services: - ZOOKEEPER_CLIENT_PORT=2181 - ZOOKEEPER_TICK_TIME=2000 hostname: zookeeper - image: kymeric/cp-zookeeper:latest + image: confluentinc/cp-zookeeper:7.2.2 ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 759caf5e1d0d2..253802e51d27e 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -62,29 +62,29 @@ services: depends_on: - mysql environment: - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_PORT=9200 - - MAE_CONSUMER_ENABLED=true - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - ELASTICSEARCH_HOST=elasticsearch - - ENTITY_SERVICE_ENABLE_RETENTION=true - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - EBEAN_DATASOURCE_USERNAME=datahub + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - EBEAN_DATASOURCE_HOST=mysql:3306 - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - EBEAN_DATASOURCE_PASSWORD=datahub - - UI_INGESTION_ENABLED=true - JAVA_OPTS=-Xms1g -Xmx1g - - GRAPH_SERVICE_IMPL=elasticsearch + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - UI_INGESTION_ENABLED=true - MCE_CONSUMER_ENABLED=true + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - ENTITY_SERVICE_ENABLE_RETENTION=true + - ELASTICSEARCH_HOST=elasticsearch - PE_CONSUMER_ENABLED=true + - MAE_CONSUMER_ENABLED=true + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - EBEAN_DATASOURCE_PASSWORD=datahub + - GRAPH_SERVICE_IMPL=elasticsearch - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - EBEAN_DATASOURCE_USERNAME=datahub hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: From 23b6f91f1da73c518d1b7725ab4d45291cbf0a38 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Tue, 24 Jan 2023 14:01:25 +0000 Subject: [PATCH 22/22] Bump confluent to 7.2.2 --- docker/docker-compose-with-cassandra.yml | 1 - docker/docker-compose-without-neo4j.yml | 1 - docker/docker-compose.yml | 1 - .../docker-compose-m1.quickstart.yml | 4 +-- ...er-compose-without-neo4j-m1.quickstart.yml | 36 +++++++++---------- ...ocker-compose-without-neo4j.quickstart.yml | 36 +++++++++---------- .../quickstart/docker-compose.quickstart.yml | 6 ++-- docker/schema-registry/env/docker.env | 10 +++--- 8 files changed, 47 insertions(+), 48 deletions(-) diff --git a/docker/docker-compose-with-cassandra.yml b/docker/docker-compose-with-cassandra.yml index 24dd3436613fe..1059ae2234d91 100644 --- a/docker/docker-compose-with-cassandra.yml +++ b/docker/docker-compose-with-cassandra.yml @@ -35,7 +35,6 @@ services: hostname: schema-registry container_name: schema-registry depends_on: - - zookeeper - broker ports: - "8081:8081" diff --git a/docker/docker-compose-without-neo4j.yml b/docker/docker-compose-without-neo4j.yml index 2c6b57b564238..a0bd0fd34c391 100644 --- a/docker/docker-compose-without-neo4j.yml +++ b/docker/docker-compose-without-neo4j.yml @@ -32,7 +32,6 @@ services: hostname: schema-registry container_name: schema-registry depends_on: - - zookeeper - broker ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b193029622f90..470746f31890d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -34,7 +34,6 @@ services: hostname: schema-registry container_name: schema-registry depends_on: - - zookeeper - broker ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml index d1b0e00850189..752e076c1c48a 100644 --- a/docker/quickstart/docker-compose-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-m1.quickstart.yml @@ -175,11 +175,11 @@ services: schema-registry: container_name: schema-registry depends_on: - - zookeeper - broker environment: - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 + - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT + - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 hostname: schema-registry image: confluentinc/cp-schema-registry:7.2.2 ports: diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index 92c0c51c9fa21..28d06b7ae2877 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -64,29 +64,29 @@ services: depends_on: - mysql environment: - - EBEAN_DATASOURCE_HOST=mysql:3306 - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - GRAPH_SERVICE_IMPL=elasticsearch - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - MAE_CONSUMER_ENABLED=true - - UI_INGESTION_ENABLED=true - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - PE_CONSUMER_ENABLED=true - - JAVA_OPTS=-Xms1g -Xmx1g - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - ELASTICSEARCH_HOST=elasticsearch - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - EBEAN_DATASOURCE_USERNAME=datahub - MCE_CONSUMER_ENABLED=true - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - EBEAN_DATASOURCE_PASSWORD=datahub + - JAVA_OPTS=-Xms1g -Xmx1g + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - UI_INGESTION_ENABLED=true - ENTITY_SERVICE_ENABLE_RETENTION=true + - MAE_CONSUMER_ENABLED=true + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - GRAPH_SERVICE_IMPL=elasticsearch + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - ELASTICSEARCH_PORT=9200 + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - EBEAN_DATASOURCE_HOST=mysql:3306 + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - PE_CONSUMER_ENABLED=true hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: @@ -153,11 +153,11 @@ services: schema-registry: container_name: schema-registry depends_on: - - zookeeper - broker environment: - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 + - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT + - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 hostname: schema-registry image: confluentinc/cp-schema-registry:7.2.2 ports: diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 253802e51d27e..177cf7f52404c 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -62,29 +62,29 @@ services: depends_on: - mysql environment: - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - EBEAN_DATASOURCE_HOST=mysql:3306 - - JAVA_OPTS=-Xms1g -Xmx1g - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - UI_INGESTION_ENABLED=true + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - GRAPH_SERVICE_IMPL=elasticsearch + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - MCE_CONSUMER_ENABLED=true - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - MAE_CONSUMER_ENABLED=true + - JAVA_OPTS=-Xms1g -Xmx1g + - EBEAN_DATASOURCE_HOST=mysql:3306 + - KAFKA_BOOTSTRAP_SERVER=broker:29092 - ENTITY_SERVICE_ENABLE_RETENTION=true + - EBEAN_DATASOURCE_USERNAME=datahub + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_HOST=elasticsearch - PE_CONSUMER_ENABLED=true - - MAE_CONSUMER_ENABLED=true - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - EBEAN_DATASOURCE_PASSWORD=datahub - - GRAPH_SERVICE_IMPL=elasticsearch - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - EBEAN_DATASOURCE_USERNAME=datahub + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: @@ -150,11 +150,11 @@ services: schema-registry: container_name: schema-registry depends_on: - - zookeeper - broker environment: - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 + - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT + - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 hostname: schema-registry image: confluentinc/cp-schema-registry:7.2.2 ports: diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index aae1bebfac681..4524570c1cb48 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -62,8 +62,8 @@ services: datahub-gms: container_name: datahub-gms depends_on: - - neo4j - mysql + - neo4j environment: - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} @@ -172,11 +172,11 @@ services: schema-registry: container_name: schema-registry depends_on: - - zookeeper - broker environment: - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 + - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT + - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 hostname: schema-registry image: confluentinc/cp-schema-registry:7.2.2 ports: diff --git a/docker/schema-registry/env/docker.env b/docker/schema-registry/env/docker.env index fbf6840116b9d..f829f418a6c8e 100644 --- a/docker/schema-registry/env/docker.env +++ b/docker/schema-registry/env/docker.env @@ -1,9 +1,11 @@ SCHEMA_REGISTRY_HOST_NAME=schemaregistry -SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 +SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT +SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 -# Uncomment to customize the Schema Registry kafka store connection -# SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT -# SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 +# Uncomment to customize the Schema Registry kafka store connection # ZOOKEEPER_SASL_ENABLED=false # KAFKA_OPTS=-Xms1g -Xmx1g # SCHEMA_REGISTRY_JMX_OPTS=-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false + +# Uncomment to use schema registry < v5.4.0 +# SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 \ No newline at end of file