fix(tests): Misc updates for tests, auth log level, and quickstart

datahub-frontend/app/client/AuthServiceClient.java

@@ -3,6 +3,8 @@
 import com.datahub.authentication.Authentication;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import java.nio.charset.StandardCharsets;
 import java.util.Objects;
 import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
@@ -150,9 +152,11 @@ public boolean signUp(@Nonnull final String userUrn, @Nonnull final String fullN
         final String jsonStr = EntityUtils.toString(entity);
         return getIsNativeUserCreatedFromJson(jsonStr);
       } else {
+        String content = response.getEntity().getContent() == null ? "" : new String(
+                response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
         throw new RuntimeException(
-            String.format("Bad response from the Metadata Service: %s %s", response.getStatusLine().toString(),
-                response.getEntity().toString()));
+            String.format("Bad response from the Metadata Service: %s %s Body: %s", response.getStatusLine().toString(),
+                response.getEntity().toString(), content));
       }
     } catch (Exception e) {
       throw new RuntimeException(String.format("Failed to create user %s", userUrn), e);

datahub-frontend/app/security/AuthenticationManager.java

@@ -51,15 +51,15 @@ public void handle(@Nonnull Callback[] callbacks) {
       NameCallback nc = null;
       PasswordCallback pc = null;
       for (Callback callback : callbacks) {
-        Logger.error("The submitted callback is of type: " + callback.getClass() + " : " + callback);
+        Logger.debug("The submitted callback is of type: " + callback.getClass() + " : " + callback);
         if (callback instanceof NameCallback) {
           nc = (NameCallback) callback;
           nc.setName(this.username);
         } else if (callback instanceof PasswordCallback) {
           pc = (PasswordCallback) callback;
           pc.setPassword(this.password.toCharArray());
         } else {
-          Logger.warn("The submitted callback is unsupported! ", callback);
+          Logger.warn("The submitted callback is unsupported! type: " + callback.getClass(), callback);
         }
       }
     }

docker/datahub-gms/env/docker.cassandra.env

@@ -3,6 +3,7 @@ KAFKA_BOOTSTRAP_SERVER=broker:29092
 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
 ELASTICSEARCH_HOST=elasticsearch
 ELASTICSEARCH_PORT=9200
+ES_BULK_REFRESH_POLICY=WAIT_UNTIL
 NEO4J_HOST=http://neo4j:7474
 NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j

docker/datahub-gms/env/docker.mariadb.env

@@ -7,6 +7,7 @@ KAFKA_BOOTSTRAP_SERVER=broker:29092
 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
 ELASTICSEARCH_HOST=elasticsearch
 ELASTICSEARCH_PORT=9200
+ES_BULK_REFRESH_POLICY=WAIT_UNTIL
 NEO4J_HOST=http://neo4j:7474
 NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j

docker/datahub-gms/env/docker.postgres.env

@@ -7,6 +7,7 @@ KAFKA_BOOTSTRAP_SERVER=broker:29092
 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
 ELASTICSEARCH_HOST=elasticsearch
 ELASTICSEARCH_PORT=9200
+ES_BULK_REFRESH_POLICY=WAIT_UNTIL
 NEO4J_HOST=http://neo4j:7474
 NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j

docker/datahub-mae-consumer/env/docker-without-neo4j.env

@@ -7,6 +7,7 @@ KAFKA_BOOTSTRAP_SERVER=broker:29092
 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
 ELASTICSEARCH_HOST=elasticsearch
 ELASTICSEARCH_PORT=9200
+ES_BULK_REFRESH_POLICY=WAIT_UNTIL
 GRAPH_SERVICE_IMPL=elasticsearch
 ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml
 

docker/datahub-mae-consumer/env/docker.env

@@ -7,6 +7,7 @@ KAFKA_BOOTSTRAP_SERVER=broker:29092
 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
 ELASTICSEARCH_HOST=elasticsearch
 ELASTICSEARCH_PORT=9200
+ES_BULK_REFRESH_POLICY=WAIT_UNTIL
 NEO4J_HOST=http://neo4j:7474
 NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j

docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml

@@ -17,6 +17,7 @@ services:
     - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
     - ELASTICSEARCH_HOST=elasticsearch
     - ELASTICSEARCH_PORT=9200
+    - ES_BULK_REFRESH_POLICY=WAIT_UNTIL
     - GRAPH_SERVICE_IMPL=elasticsearch
     - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml
     hostname: datahub-mae-consumer

docker/quickstart/docker-compose.consumers.quickstart.yml

@@ -18,6 +18,7 @@ services:
     - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
     - ELASTICSEARCH_HOST=elasticsearch
     - ELASTICSEARCH_PORT=9200
+    - ES_BULK_REFRESH_POLICY=WAIT_UNTIL
     - NEO4J_HOST=http://neo4j:7474
     - NEO4J_URI=bolt://neo4j
     - NEO4J_USERNAME=neo4j

metadata-integration/java/spark-lineage/spark-smoke-test/smoke.sh

@@ -40,7 +40,9 @@ echo "--------------------------------------------------------------------"
 
 pwd ../../../
 
-../../../../docker/dev.sh -d
+DATAHUB_TELEMETRY_ENABLED=false  \
+DOCKER_COMPOSE_BASE="file://$(cd "$(dirname "${DIR}/../../../../../")"; pwd)" \
+datahub docker quickstart --build-locally --dump-logs-on-failure
 is_healthy "datahub-gms" 60
 
 echo "--------------------------------------------------------------------"

smoke-test/tests/browse/browse_test.py

@@ -51,7 +51,7 @@ def test_get_browse_paths(frontend_session, ingest_cleanup_data):
     # /prod -- There should be one entity
     get_browse_paths_json = {
         "query": get_browse_paths_query,
-        "variables": {"input": { "type": "DATASET", "path": ["prod"], "start": 0, "count": 10 } },
+        "variables": {"input": { "type": "DATASET", "path": ["prod"], "start": 0, "count": 100 } },
     }
 
     response = frontend_session.post(

smoke-test/tests/utils.py

@@ -72,13 +72,8 @@ def is_k8s_enabled():
 
 
 def wait_for_healthcheck_util():
-    if is_k8s_enabled():
-        # Simply assert that kubernetes endpoints are healthy, but don't wait.
-        assert not check_endpoint(f"{get_frontend_url()}/admin")
-        assert not check_endpoint(f"{get_gms_url()}/health")
-    else:
-        # Simply assert that docker is healthy, but don't wait.
-        assert not check_local_docker_containers()
+    assert not check_endpoint(f"{get_frontend_url()}/admin")
+    assert not check_endpoint(f"{get_gms_url()}/health")
 
 
 def check_endpoint(url):