Merge branch 'datahub-project:master' into master

acryldata · Oct 19, 2022 · 8108f96 · 8108f96
2 parents aa718b8 + 98eba90
commit 8108f96
Show file tree

Hide file tree

Showing 17 changed files with 282 additions and 84 deletions.
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -355,7 +355,7 @@ jobs:
   smoke_test:
     name: Run Smoke Tests
     runs-on: ubuntu-latest
-    needs: [setup,gms_build,frontend_build,kafka_setup_build,mysql_setup_build,elasticsearch_setup_build]
+    needs: [setup,gms_build,frontend_build,kafka_setup_build,mysql_setup_build,elasticsearch_setup_build,mae_consumer_build,mce_consumer_build]
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2

diff --git a/datahub-web-react/src/app/analytics/event.ts b/datahub-web-react/src/app/analytics/event.ts
@@ -41,6 +41,12 @@ export enum EventType {
     ActivatePolicyEvent,
     ShowSimplifiedHomepageEvent,
     ShowStandardHomepageEvent,
+    CreateGlossaryEntityEvent,
+    CreateDomainEvent,
+    CreateIngestionSourceEvent,
+    UpdateIngestionSourceEvent,
+    DeleteIngestionSourceEvent,
+    ExecuteIngestionSourceEvent,
 }
 
 /**
@@ -302,6 +308,8 @@ export interface BatchSelectUserRoleEvent extends BaseEvent {
     userUrns: string[];
 }
 
+// Policy events
+
 export interface CreatePolicyEvent extends BaseEvent {
     type: EventType.CreatePolicyEvent;
 }
@@ -329,6 +337,38 @@ export interface ShowStandardHomepageEvent extends BaseEvent {
     type: EventType.ShowStandardHomepageEvent;
 }
 
+// Business glossary events
+
+export interface CreateGlossaryEntityEvent extends BaseEvent {
+    type: EventType.CreateGlossaryEntityEvent;
+    entityType: EntityType;
+    parentNodeUrn?: string;
+}
+
+export interface CreateDomainEvent extends BaseEvent {
+    type: EventType.CreateDomainEvent;
+}
+
+export interface CreateIngestionSourceEvent extends BaseEvent {
+    type: EventType.CreateIngestionSourceEvent;
+    sourceType: string;
+    interval?: string;
+}
+
+export interface UpdateIngestionSourceEvent extends BaseEvent {
+    type: EventType.UpdateIngestionSourceEvent;
+    sourceType: string;
+    interval?: string;
+}
+
+export interface DeleteIngestionSourceEvent extends BaseEvent {
+    type: EventType.DeleteIngestionSourceEvent;
+}
+
+export interface ExecuteIngestionSourceEvent extends BaseEvent {
+    type: EventType.ExecuteIngestionSourceEvent;
+}
+
 /**
  * Event consisting of a union of specific event types.
  */
@@ -368,4 +408,10 @@ export type Event =
     | DeactivatePolicyEvent
     | ActivatePolicyEvent
     | ShowSimplifiedHomepageEvent
-    | ShowStandardHomepageEvent;
+    | ShowStandardHomepageEvent
+    | CreateGlossaryEntityEvent
+    | CreateDomainEvent
+    | CreateIngestionSourceEvent
+    | UpdateIngestionSourceEvent
+    | DeleteIngestionSourceEvent
+    | ExecuteIngestionSourceEvent;
diff --git a/datahub-web-react/src/app/domain/CreateDomainModal.tsx b/datahub-web-react/src/app/domain/CreateDomainModal.tsx
@@ -4,6 +4,7 @@ import { message, Button, Input, Modal, Typography, Form, Collapse, Tag } from '
 import { useCreateDomainMutation } from '../../graphql/domain.generated';
 import { useEnterKeyListener } from '../shared/useEnterKeyListener';
 import { groupIdTextValidation } from '../shared/textUtil';
+import analytics, { EventType } from '../analytics';
 
 const SuggestedNamesGroup = styled.div`
     margin-top: 12px;
@@ -40,6 +41,13 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) {
                 },
             },
         })
+            .then(({ errors }) => {
+                if (!errors) {
+                    analytics.event({
+                        type: EventType.CreateDomainEvent,
+                    });
+                }
+            })
             .catch((e) => {
                 message.destroy();
                 message.error({ content: `Failed to create Domain!: \n ${e.message || ''}`, duration: 3 });

diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/CreateGlossaryEntityModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/CreateGlossaryEntityModal.tsx
@@ -9,6 +9,7 @@ import { EntityType } from '../../../../types.generated';
 import { useEntityRegistry } from '../../../useEntityRegistry';
 import NodeParentSelect from './NodeParentSelect';
 import { useEntityData, useRefetch } from '../EntityContext';
+import analytics, { EventType } from '../../../analytics';
 
 const StyledItem = styled(Form.Item)`
     margin-bottom: 0;
@@ -52,6 +53,11 @@ function CreateGlossaryEntityModal(props: Props) {
             .then(() => {
                 message.loading({ content: 'Updating...', duration: 2 });
                 setTimeout(() => {
+                    analytics.event({
+                        type: EventType.CreateGlossaryEntityEvent,
+                        entityType,
+                        parentNodeUrn: selectedParentUrn || undefined,
+                    });
                     message.success({
                         content: `Created ${entityRegistry.getEntityName(entityType)}!`,
                         duration: 2,

diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
@@ -25,6 +25,7 @@ import IngestionSourceTable from './IngestionSourceTable';
 import { scrollToTop } from '../../shared/searchUtils';
 import useRefreshIngestionData from './executions/useRefreshIngestionData';
 import { isExecutionRequestActive } from './executions/IngestionSourceExecutionList';
+import analytics, { EventType } from '../../analytics';
 
 const SourceContainer = styled.div``;
 
@@ -141,6 +142,9 @@ export const IngestionSourceList = () => {
             },
         })
             .then(() => {
+                analytics.event({
+                    type: EventType.ExecuteIngestionSourceEvent,
+                });
                 message.success({
                     content: `Successfully submitted ingestion execution request!`,
                     duration: 3,
@@ -171,6 +175,11 @@ export const IngestionSourceList = () => {
             // Update:
             updateIngestionSource({ variables: { urn: focusSourceUrn as string, input } })
                 .then(() => {
+                    analytics.event({
+                        type: EventType.UpdateIngestionSourceEvent,
+                        sourceType: input.type,
+                        interval: input.schedule?.interval,
+                    });
                     message.success({
                         content: `Successfully updated ingestion source!`,
                         duration: 3,
@@ -193,6 +202,11 @@ export const IngestionSourceList = () => {
                     message.loading({ content: 'Loading...', duration: 2 });
                     setTimeout(() => {
                         refetch();
+                        analytics.event({
+                            type: EventType.CreateIngestionSourceEvent,
+                            sourceType: input.type,
+                            interval: input.schedule?.interval,
+                        });
                         message.success({
                             content: `Successfully created ingestion source!`,
                             duration: 3,
@@ -226,6 +240,9 @@ export const IngestionSourceList = () => {
             variables: { urn },
         })
             .then(() => {
+                analytics.event({
+                    type: EventType.DeleteIngestionSourceEvent,
+                });
                 message.success({ content: 'Removed ingestion source.', duration: 2 });
                 const newRemovedUrns = [...removedUrns, urn];
                 setRemovedUrns(newRemovedUrns);

diff --git a/docker/docker-compose.consumers-without-neo4j.yml b/docker/docker-compose.consumers-without-neo4j.yml
@@ -10,7 +10,7 @@ services:
     build:
       context: ../
       dockerfile: docker/datahub-mae-consumer/Dockerfile
-    image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-head}
+    image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head}
     env_file: datahub-mae-consumer/env/docker-without-neo4j.env
     hostname: datahub-mae-consumer
     container_name: datahub-mae-consumer
@@ -24,7 +24,7 @@ services:
     build:
       context: ../
       dockerfile: docker/datahub-mce-consumer/Dockerfile
-    image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-head}
+    image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head}
     env_file: datahub-mce-consumer/env/docker.env
     hostname: datahub-mce-consumer
     container_name: datahub-mce-consumer

diff --git a/docker/docker-compose.consumers.yml b/docker/docker-compose.consumers.yml
@@ -10,7 +10,7 @@ services:
     build:
       context: ../
       dockerfile: docker/datahub-mae-consumer/Dockerfile
-    image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-head}
+    image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head}
     env_file: datahub-mae-consumer/env/docker.env
     hostname: datahub-mae-consumer
     container_name: datahub-mae-consumer
@@ -25,7 +25,7 @@ services:
     build:
       context: ../
       dockerfile: docker/datahub-mce-consumer/Dockerfile
-    image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-head}
+    image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head}
     env_file: datahub-mce-consumer/env/docker.env
     hostname: datahub-mce-consumer
     container_name: datahub-mce-consumer

diff --git a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml
@@ -20,7 +20,7 @@ services:
     - GRAPH_SERVICE_IMPL=elasticsearch
     - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml
     hostname: datahub-mae-consumer
-    image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-head}
+    image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head}
     ports:
     - 9091:9091
   datahub-mce-consumer:
@@ -35,7 +35,7 @@ services:
     - DATAHUB_GMS_HOST=datahub-gms
     - DATAHUB_GMS_PORT=8080
     hostname: datahub-mce-consumer
-    image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-head}
+    image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head}
     ports:
     - 9090:9090
 version: '2.3'
diff --git a/docker/quickstart/docker-compose.consumers.quickstart.yml b/docker/quickstart/docker-compose.consumers.quickstart.yml
@@ -25,7 +25,7 @@ services:
     - GRAPH_SERVICE_IMPL=neo4j
     - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml
     hostname: datahub-mae-consumer
-    image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-head}
+    image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head}
     ports:
     - 9091:9091
   datahub-mce-consumer:
@@ -40,7 +40,7 @@ services:
     - DATAHUB_GMS_HOST=datahub-gms
     - DATAHUB_GMS_PORT=8080
     hostname: datahub-mce-consumer
-    image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-head}
+    image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head}
     ports:
     - 9090:9090
 version: '2.3'
diff --git a/docs/quickstart.md b/docs/quickstart.md
@@ -118,6 +118,7 @@ docker system prune
 <summary>
 Still stuck?
 </summary>
+
 Hop over to our [Slack community](https://slack.datahubproject.io) and ask for help in the [#troubleshoot](https://datahubspace.slack.com/archives/C029A3M079U) channel!
 </details>
 

diff --git a/docs/what/mxe.md b/docs/what/mxe.md
@@ -6,7 +6,7 @@ DataHub makes use a few important Kafka events for operation. The most notable o
 2. Metadata Change Log (Versioned + Timeseries) 
 3. Platform Event
 
-Each event is originally authored using [PDL]( https://linkedin.github.io/rest.li/DATA-Data-Schema-and-Templates), a modeling language developed by LinkedIn, and 
+Each event is originally authored using [PDL](https://linkedin.github.io/rest.li/pdl_schema), a modeling language developed by LinkedIn, and 
 then converted into their Avro equivalents, which are used when writing and reading the events to Kafka. 
 
 In the document, we'll describe each of these events in detail - including notes about their structure & semantics. 

diff --git a/metadata-ingestion/src/datahub/utilities/memory_footprint.py b/metadata-ingestion/src/datahub/utilities/memory_footprint.py
@@ -1,7 +1,7 @@
 from collections import deque
 from itertools import chain
 from sys import getsizeof
-from typing import Any, Dict
+from typing import Any, Callable
 
 
 def total_size(o: Any, handlers: Any = {}) -> int:
@@ -15,8 +15,7 @@ def total_size(o: Any, handlers: Any = {}) -> int:
     Based on https://github.com/ActiveState/recipe-577504-compute-mem-footprint/blob/master/recipe.py
     """
 
-    def dict_handler(d: Dict) -> chain[Any]:
-        return chain.from_iterable(d.items())
+    dict_handler: Callable[[Any], chain[Any]] = lambda d: chain.from_iterable(d.items())
 
     all_handlers = {
         tuple: iter,

diff --git a/metadata-ingestion/tests/unit/utilities/test_memory_footprint.py b/metadata-ingestion/tests/unit/utilities/test_memory_footprint.py
@@ -0,0 +1,27 @@
+from collections import defaultdict
+
+from datahub.utilities import memory_footprint
+
+
+def test_total_size_with_empty_dict():
+    size = memory_footprint.total_size({})
+    # Only asserting if it is bigger than 0 because the actual sizes differs per python version
+    assert size > 0
+
+
+def test_total_size_with_list():
+    size = memory_footprint.total_size({"1": [1, 2, 3, 4]})
+    # Only asserting if it is bigger than 0 because the actual sizes differs per python version
+    assert size > 0
+
+
+def test_total_size_with_none():
+    size = memory_footprint.total_size(None)
+    # Only asserting if it is bigger than 0 because the actual sizes differs per python version
+    assert size > 0
+
+
+def test_total_size_with_defaultdict():
+    size = memory_footprint.total_size(defaultdict)
+    # Only asserting if it is bigger than 0 because the actual sizes differs per python version
+    assert size > 0
diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/telemetry/TrackingService.java b/metadata-service/auth-impl/src/main/java/com/datahub/telemetry/TrackingService.java
@@ -53,12 +53,14 @@ public class TrackingService {
   private static final String DURATION_FIELD = "duration";
   private static final String ROLE_URN_FIELD = "roleUrn";
   private static final String POLICY_URN_FIELD = "policyUrn";
+  private static final String SOURCE_TYPE_FIELD = "sourceType";
+  private static final String INTERVAL_FIELD = "interval";
 
   private static final Set<String> ALLOWED_EVENT_FIELDS = new HashSet<>(
       ImmutableList.of(EVENT_TYPE_FIELD, SIGN_UP_TITLE_FIELD, ENTITY_TYPE_FIELD, ENTITY_TYPE_FILTER_FIELD,
           PAGE_NUMBER_FIELD, PAGE_FIELD, TOTAL_FIELD, INDEX_FIELD, RESULT_TYPE_FIELD, RENDER_ID_FIELD, MODULE_ID_FIELD,
           RENDER_TYPE_FIELD, SCENARIO_TYPE_FIELD, SECTION_FIELD, ACCESS_TOKEN_TYPE_FIELD, DURATION_FIELD,
-          ROLE_URN_FIELD, POLICY_URN_FIELD));
+          ROLE_URN_FIELD, POLICY_URN_FIELD, SOURCE_TYPE_FIELD, INTERVAL_FIELD));
 
   private static final String ACTOR_URN_FIELD = "actorUrn";
   private static final String ORIGIN_FIELD = "origin";
@@ -69,9 +71,10 @@ public class TrackingService {
   private static final String PATH_FIELD = "path";
   private static final String USER_URN_FIELD = "userUrn";
   private static final String USER_URNS_FIELD = "userUrns";
+  private static final String PARENT_NODE_URN_FIELD = "parentNodeUrn";
   private static final Set<String> ALLOWED_OBFUSCATED_EVENT_FIELDS = new HashSet<>(
       ImmutableList.of(ACTOR_URN_FIELD, ORIGIN_FIELD, ENTITY_URN_FIELD, ENTITY_URNS_FIELD, GROUP_NAME_FIELD,
-          SECTION_FIELD, ENTITY_PAGE_FILTER_FIELD, PATH_FIELD, USER_URN_FIELD, USER_URNS_FIELD));
+          SECTION_FIELD, ENTITY_PAGE_FILTER_FIELD, PATH_FIELD, USER_URN_FIELD, USER_URNS_FIELD, PARENT_NODE_URN_FIELD));
 
   private final MixpanelAPI _mixpanelAPI;
   private final MessageBuilder _mixpanelMessageBuilder;

diff --git a/smoke-test/tests/cypress/cypress/integration/containers/containers.js b/smoke-test/tests/cypress/cypress/integration/containers/containers.js
@@ -0,0 +1,16 @@
+describe("containers", () => {
+  it("can see elements inside the container", () => {
+    cy.login();
+    cy.visit(
+      "http://localhost:9002/container/urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb"
+    );
+
+    cy.contains("jaffle_shop");
+
+    cy.contains("customers");
+    cy.contains("customers_source");
+    cy.contains("orders");
+    cy.contains("raw_orders");
+    cy.contains("1 - 9 of 9");
+  });
+});
diff --git a/smoke-test/tests/cypress/cypress/integration/domains/domains.js b/smoke-test/tests/cypress/cypress/integration/domains/domains.js
@@ -0,0 +1,12 @@
+describe("domains", () => {
+  it("can see elements inside the domain", () => {
+    cy.login();
+    cy.visit(
+      "http://localhost:9002/domain/urn:li:domain:marketing/Entities?is_lineage_mode=false"
+    );
+
+    cy.contains("Marketing");
+    cy.contains("SampleCypressKafkaDataset");
+    cy.contains("1 - 1 of 1");
+  });
+});