From abe6781e252e1d7bb89ad9c13de1c197d554d46a Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Tue, 18 Oct 2022 07:33:02 -0700 Subject: [PATCH 1/6] adding domain & container tests (#6221) --- .../integration/containers/containers.js | 16 ++ .../cypress/integration/domains/domains.js | 12 ++ smoke-test/tests/cypress/data.json | 199 ++++++++++++------ 3 files changed, 159 insertions(+), 68 deletions(-) create mode 100644 smoke-test/tests/cypress/cypress/integration/containers/containers.js create mode 100644 smoke-test/tests/cypress/cypress/integration/domains/domains.js diff --git a/smoke-test/tests/cypress/cypress/integration/containers/containers.js b/smoke-test/tests/cypress/cypress/integration/containers/containers.js new file mode 100644 index 00000000000000..1b323ece1fd739 --- /dev/null +++ b/smoke-test/tests/cypress/cypress/integration/containers/containers.js @@ -0,0 +1,16 @@ +describe("containers", () => { + it("can see elements inside the container", () => { + cy.login(); + cy.visit( + "http://localhost:9002/container/urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb" + ); + + cy.contains("jaffle_shop"); + + cy.contains("customers"); + cy.contains("customers_source"); + cy.contains("orders"); + cy.contains("raw_orders"); + cy.contains("1 - 9 of 9"); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/integration/domains/domains.js b/smoke-test/tests/cypress/cypress/integration/domains/domains.js new file mode 100644 index 00000000000000..8f3cc86f88ae9b --- /dev/null +++ b/smoke-test/tests/cypress/cypress/integration/domains/domains.js @@ -0,0 +1,12 @@ +describe("domains", () => { + it("can see elements inside the domain", () => { + cy.login(); + cy.visit( + "http://localhost:9002/domain/urn:li:domain:marketing/Entities?is_lineage_mode=false" + ); + + cy.contains("Marketing"); + cy.contains("SampleCypressKafkaDataset"); + cy.contains("1 - 1 of 1"); + }); +}); diff --git a/smoke-test/tests/cypress/data.json b/smoke-test/tests/cypress/data.json index 678bab5d269522..ebc96d5744c060 100644 --- a/smoke-test/tests/cypress/data.json +++ b/smoke-test/tests/cypress/data.json @@ -224,7 +224,18 @@ { "fieldPath": "shipment_info", "globalTags": { "tags": [{ "tag": "urn:li:tag:Legacy" }] }, - "glossaryTerms": { "terms": [{ "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" }], "auditStamp": { "time": 0, "actor": "urn:li:corpuser:jdoe", "impersonator": null }} + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" + } + ], + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } } ] } @@ -623,7 +634,18 @@ }, "nativeDataType": "boolean", "recursive": false, - "glossaryTerms": { "terms": [{ "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" }], "auditStamp": { "time": 0, "actor": "urn:li:corpuser:jdoe", "impersonator": null }} + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CypressNode.CypressColumnInfoType" + } + ], + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } }, { "fieldPath": "timestamp", @@ -792,16 +814,18 @@ ], "primaryKeys": null, "foreignKeysSpecs": null, - "foreignKeys": [{ - "name": "user id", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" - }] + "foreignKeys": [ + { + "name": "user id", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" + } + ] } } ] @@ -934,16 +958,18 @@ ], "primaryKeys": null, "foreignKeysSpecs": null, - "foreignKeys": [{ - "name": "user id", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" - }] + "foreignKeys": [ + { + "name": "user id", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" + } + ] } }, { @@ -1150,16 +1176,18 @@ ], "primaryKeys": ["user_name"], "foreignKeysSpecs": null, - "foreignKeys": [{ - "name": "user session", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" - }] + "foreignKeys": [ + { + "name": "user session", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_deleted,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)" + } + ] } }, { @@ -1484,10 +1512,12 @@ }, { "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [{ - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - }], + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], "lastModified": { "time": 1581407189000, "actor": "urn:li:corpuser:jdoe" @@ -1504,17 +1534,20 @@ "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryNodeSnapshot": { "urn": "urn:li:glossaryNode:CypressNode", - "aspects": [{ - "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { - "definition": "Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities" - } - }, + "aspects": [ + { + "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "definition": "Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities" + } + }, { "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [{ - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - }], + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], "lastModified": { "time": 1581407189000, "actor": "urn:li:corpuser:jdoe" @@ -1526,7 +1559,7 @@ }, "proposedDelta": null }, - { + { "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { @@ -1567,8 +1600,8 @@ "changeType": "UPSERT", "aspectName": "operation", "aspect": { - "value": "{\"timestampMillis\": 1622243780153, \"lastUpdatedTimestamp\": 1622243780153, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": []}", - "contentType": "application/json" + "value": "{\"timestampMillis\": 1622243780153, \"lastUpdatedTimestamp\": 1622243780153, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"affectedDatasets\": []}", + "contentType": "application/json" }, "systemMetadata": null }, @@ -1584,7 +1617,8 @@ "contentType": "application/json" }, "systemMetadata": null - },{ + }, + { "auditHeader": null, "entityType": "dataProcessInstance", "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", @@ -1596,7 +1630,8 @@ "contentType": "application/json" }, "systemMetadata": null - },{ + }, + { "auditHeader": null, "entityType": "dataProcessInstance", "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", @@ -1608,7 +1643,8 @@ "contentType": "application/json" }, "systemMetadata": null - },{ + }, + { "auditHeader": null, "entityType": "dataProcessInstance", "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", @@ -1620,7 +1656,8 @@ "contentType": "application/json" }, "systemMetadata": null - },{ + }, + { "auditHeader": null, "entityType": "dataProcessInstance", "entityUrn": "urn:li:dataProcessInstance:c39d6e424c70a1b3e8982535c68b5b5888", @@ -1644,7 +1681,9 @@ "description": null, "dataType": "TEXT", "version": null, - "sources": ["urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)"] + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)" + ] } }, { @@ -1669,7 +1708,9 @@ "description": "this is a description from source system", "dataType": "ORDINAL", "version": null, - "sources": ["urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)"] + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)" + ] } }, { @@ -1691,9 +1732,7 @@ "aspects": [ { "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/sagemaker/cypress-feature-table" - ] + "paths": ["/sagemaker/cypress-feature-table"] } }, { @@ -1752,7 +1791,9 @@ } ], "onlineMetrics": null, - "mlFeatures": ["urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)"], + "mlFeatures": [ + "urn:li:mlFeature:(cypress-test-2,some-cypress-feature-1)" + ], "tags": [], "deployments": [], "trainingJobs": [], @@ -1764,9 +1805,7 @@ }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/sagemaker/cypress-model-package-group/cypress-model" - ] + "paths": ["/sagemaker/cypress-model-package-group/cypress-model"] } } ] @@ -1812,9 +1851,7 @@ }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/sagemaker/cypress-model-package-group" - ] + "paths": ["/sagemaker/cypress-model-package-group"] } } ] @@ -1843,10 +1880,12 @@ }, { "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [{ - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - }], + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], "lastModified": { "time": 1581407189000, "actor": "urn:li:corpuser:jdoe" @@ -1857,5 +1896,29 @@ } }, "proposedDelta": null + }, + { + "auditHeader": null, + "entityType": "domain", + "entityUrn": "urn:li:domain:marketing", + "changeType": "UPSERT", + "aspectName": "domainProperties", + "aspect": { + "value": "{\"name\": \"Marketing\", \"description\": \"My custom domain\" }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "value": "{\"domains\": [\"urn:li:domain:marketing\"] }", + "contentType": "application/json" + }, + "systemMetadata": null } ] From a3db265e41b3d8bed29cd2d8d02850149a4d7ab3 Mon Sep 17 00:00:00 2001 From: Aditya Radhakrishnan Date: Tue, 18 Oct 2022 09:32:00 -0700 Subject: [PATCH 2/6] fix(docs): fix pdl link for mxe docs (#6230) --- docs/what/mxe.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/what/mxe.md b/docs/what/mxe.md index 0cbd84a5ad2d67..8af96360858a33 100644 --- a/docs/what/mxe.md +++ b/docs/what/mxe.md @@ -6,7 +6,7 @@ DataHub makes use a few important Kafka events for operation. The most notable o 2. Metadata Change Log (Versioned + Timeseries) 3. Platform Event -Each event is originally authored using [PDL]( https://linkedin.github.io/rest.li/DATA-Data-Schema-and-Templates), a modeling language developed by LinkedIn, and +Each event is originally authored using [PDL](https://linkedin.github.io/rest.li/pdl_schema), a modeling language developed by LinkedIn, and then converted into their Avro equivalents, which are used when writing and reading the events to Kafka. In the document, we'll describe each of these events in detail - including notes about their structure & semantics. From 2a501dead11689c870089e5f1519dd64e7e0f99b Mon Sep 17 00:00:00 2001 From: Aditya Radhakrishnan Date: Tue, 18 Oct 2022 15:48:10 -0700 Subject: [PATCH 3/6] feat(telemetry): add telemetry events to the glossary, domains, and managed ingestion pages (#6216) Co-authored-by: John Joyce --- datahub-web-react/src/app/analytics/event.ts | 48 ++++++++++++++++++- .../src/app/domain/CreateDomainModal.tsx | 8 ++++ .../CreateGlossaryEntityModal.tsx | 6 +++ .../app/ingest/source/IngestionSourceList.tsx | 17 +++++++ .../datahub/telemetry/TrackingService.java | 7 ++- 5 files changed, 83 insertions(+), 3 deletions(-) diff --git a/datahub-web-react/src/app/analytics/event.ts b/datahub-web-react/src/app/analytics/event.ts index 0f4ffe0c0a6af4..dbcebf3742edfa 100644 --- a/datahub-web-react/src/app/analytics/event.ts +++ b/datahub-web-react/src/app/analytics/event.ts @@ -41,6 +41,12 @@ export enum EventType { ActivatePolicyEvent, ShowSimplifiedHomepageEvent, ShowStandardHomepageEvent, + CreateGlossaryEntityEvent, + CreateDomainEvent, + CreateIngestionSourceEvent, + UpdateIngestionSourceEvent, + DeleteIngestionSourceEvent, + ExecuteIngestionSourceEvent, } /** @@ -302,6 +308,8 @@ export interface BatchSelectUserRoleEvent extends BaseEvent { userUrns: string[]; } +// Policy events + export interface CreatePolicyEvent extends BaseEvent { type: EventType.CreatePolicyEvent; } @@ -329,6 +337,38 @@ export interface ShowStandardHomepageEvent extends BaseEvent { type: EventType.ShowStandardHomepageEvent; } +// Business glossary events + +export interface CreateGlossaryEntityEvent extends BaseEvent { + type: EventType.CreateGlossaryEntityEvent; + entityType: EntityType; + parentNodeUrn?: string; +} + +export interface CreateDomainEvent extends BaseEvent { + type: EventType.CreateDomainEvent; +} + +export interface CreateIngestionSourceEvent extends BaseEvent { + type: EventType.CreateIngestionSourceEvent; + sourceType: string; + interval?: string; +} + +export interface UpdateIngestionSourceEvent extends BaseEvent { + type: EventType.UpdateIngestionSourceEvent; + sourceType: string; + interval?: string; +} + +export interface DeleteIngestionSourceEvent extends BaseEvent { + type: EventType.DeleteIngestionSourceEvent; +} + +export interface ExecuteIngestionSourceEvent extends BaseEvent { + type: EventType.ExecuteIngestionSourceEvent; +} + /** * Event consisting of a union of specific event types. */ @@ -368,4 +408,10 @@ export type Event = | DeactivatePolicyEvent | ActivatePolicyEvent | ShowSimplifiedHomepageEvent - | ShowStandardHomepageEvent; + | ShowStandardHomepageEvent + | CreateGlossaryEntityEvent + | CreateDomainEvent + | CreateIngestionSourceEvent + | UpdateIngestionSourceEvent + | DeleteIngestionSourceEvent + | ExecuteIngestionSourceEvent; diff --git a/datahub-web-react/src/app/domain/CreateDomainModal.tsx b/datahub-web-react/src/app/domain/CreateDomainModal.tsx index 3306104c1698d8..7338ca223893ec 100644 --- a/datahub-web-react/src/app/domain/CreateDomainModal.tsx +++ b/datahub-web-react/src/app/domain/CreateDomainModal.tsx @@ -4,6 +4,7 @@ import { message, Button, Input, Modal, Typography, Form, Collapse, Tag } from ' import { useCreateDomainMutation } from '../../graphql/domain.generated'; import { useEnterKeyListener } from '../shared/useEnterKeyListener'; import { groupIdTextValidation } from '../shared/textUtil'; +import analytics, { EventType } from '../analytics'; const SuggestedNamesGroup = styled.div` margin-top: 12px; @@ -40,6 +41,13 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { }, }, }) + .then(({ errors }) => { + if (!errors) { + analytics.event({ + type: EventType.CreateDomainEvent, + }); + } + }) .catch((e) => { message.destroy(); message.error({ content: `Failed to create Domain!: \n ${e.message || ''}`, duration: 3 }); diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/CreateGlossaryEntityModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/CreateGlossaryEntityModal.tsx index f136ea76d7dc77..ebbd1843d4b512 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/CreateGlossaryEntityModal.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/CreateGlossaryEntityModal.tsx @@ -9,6 +9,7 @@ import { EntityType } from '../../../../types.generated'; import { useEntityRegistry } from '../../../useEntityRegistry'; import NodeParentSelect from './NodeParentSelect'; import { useEntityData, useRefetch } from '../EntityContext'; +import analytics, { EventType } from '../../../analytics'; const StyledItem = styled(Form.Item)` margin-bottom: 0; @@ -52,6 +53,11 @@ function CreateGlossaryEntityModal(props: Props) { .then(() => { message.loading({ content: 'Updating...', duration: 2 }); setTimeout(() => { + analytics.event({ + type: EventType.CreateGlossaryEntityEvent, + entityType, + parentNodeUrn: selectedParentUrn || undefined, + }); message.success({ content: `Created ${entityRegistry.getEntityName(entityType)}!`, duration: 2, diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx index adf466b9e2a173..2700a53bd19177 100644 --- a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx +++ b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx @@ -25,6 +25,7 @@ import IngestionSourceTable from './IngestionSourceTable'; import { scrollToTop } from '../../shared/searchUtils'; import useRefreshIngestionData from './executions/useRefreshIngestionData'; import { isExecutionRequestActive } from './executions/IngestionSourceExecutionList'; +import analytics, { EventType } from '../../analytics'; const SourceContainer = styled.div``; @@ -141,6 +142,9 @@ export const IngestionSourceList = () => { }, }) .then(() => { + analytics.event({ + type: EventType.ExecuteIngestionSourceEvent, + }); message.success({ content: `Successfully submitted ingestion execution request!`, duration: 3, @@ -171,6 +175,11 @@ export const IngestionSourceList = () => { // Update: updateIngestionSource({ variables: { urn: focusSourceUrn as string, input } }) .then(() => { + analytics.event({ + type: EventType.UpdateIngestionSourceEvent, + sourceType: input.type, + interval: input.schedule?.interval, + }); message.success({ content: `Successfully updated ingestion source!`, duration: 3, @@ -193,6 +202,11 @@ export const IngestionSourceList = () => { message.loading({ content: 'Loading...', duration: 2 }); setTimeout(() => { refetch(); + analytics.event({ + type: EventType.CreateIngestionSourceEvent, + sourceType: input.type, + interval: input.schedule?.interval, + }); message.success({ content: `Successfully created ingestion source!`, duration: 3, @@ -226,6 +240,9 @@ export const IngestionSourceList = () => { variables: { urn }, }) .then(() => { + analytics.event({ + type: EventType.DeleteIngestionSourceEvent, + }); message.success({ content: 'Removed ingestion source.', duration: 2 }); const newRemovedUrns = [...removedUrns, urn]; setRemovedUrns(newRemovedUrns); diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/telemetry/TrackingService.java b/metadata-service/auth-impl/src/main/java/com/datahub/telemetry/TrackingService.java index 3147d7d6192fc6..0294c9e6d37445 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/telemetry/TrackingService.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/telemetry/TrackingService.java @@ -53,12 +53,14 @@ public class TrackingService { private static final String DURATION_FIELD = "duration"; private static final String ROLE_URN_FIELD = "roleUrn"; private static final String POLICY_URN_FIELD = "policyUrn"; + private static final String SOURCE_TYPE_FIELD = "sourceType"; + private static final String INTERVAL_FIELD = "interval"; private static final Set ALLOWED_EVENT_FIELDS = new HashSet<>( ImmutableList.of(EVENT_TYPE_FIELD, SIGN_UP_TITLE_FIELD, ENTITY_TYPE_FIELD, ENTITY_TYPE_FILTER_FIELD, PAGE_NUMBER_FIELD, PAGE_FIELD, TOTAL_FIELD, INDEX_FIELD, RESULT_TYPE_FIELD, RENDER_ID_FIELD, MODULE_ID_FIELD, RENDER_TYPE_FIELD, SCENARIO_TYPE_FIELD, SECTION_FIELD, ACCESS_TOKEN_TYPE_FIELD, DURATION_FIELD, - ROLE_URN_FIELD, POLICY_URN_FIELD)); + ROLE_URN_FIELD, POLICY_URN_FIELD, SOURCE_TYPE_FIELD, INTERVAL_FIELD)); private static final String ACTOR_URN_FIELD = "actorUrn"; private static final String ORIGIN_FIELD = "origin"; @@ -69,9 +71,10 @@ public class TrackingService { private static final String PATH_FIELD = "path"; private static final String USER_URN_FIELD = "userUrn"; private static final String USER_URNS_FIELD = "userUrns"; + private static final String PARENT_NODE_URN_FIELD = "parentNodeUrn"; private static final Set ALLOWED_OBFUSCATED_EVENT_FIELDS = new HashSet<>( ImmutableList.of(ACTOR_URN_FIELD, ORIGIN_FIELD, ENTITY_URN_FIELD, ENTITY_URNS_FIELD, GROUP_NAME_FIELD, - SECTION_FIELD, ENTITY_PAGE_FILTER_FIELD, PATH_FIELD, USER_URN_FIELD, USER_URNS_FIELD)); + SECTION_FIELD, ENTITY_PAGE_FILTER_FIELD, PATH_FIELD, USER_URN_FIELD, USER_URNS_FIELD, PARENT_NODE_URN_FIELD)); private final MixpanelAPI _mixpanelAPI; private final MessageBuilder _mixpanelMessageBuilder; From 9015a43f258dc79e1b0f78f0ac9a8dbce75c5ac9 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Wed, 19 Oct 2022 02:59:31 +0200 Subject: [PATCH 4/6] fix(ingest): bigquery-beta - Adding python 3.8 fix for memory footprint util (#6228) --- .../src/datahub/utilities/memory_footprint.py | 5 ++-- .../unit/utilities/test_memory_footprint.py | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 metadata-ingestion/tests/unit/utilities/test_memory_footprint.py diff --git a/metadata-ingestion/src/datahub/utilities/memory_footprint.py b/metadata-ingestion/src/datahub/utilities/memory_footprint.py index 9fc1900441426a..fa9e64cbf47380 100644 --- a/metadata-ingestion/src/datahub/utilities/memory_footprint.py +++ b/metadata-ingestion/src/datahub/utilities/memory_footprint.py @@ -1,7 +1,7 @@ from collections import deque from itertools import chain from sys import getsizeof -from typing import Any, Dict +from typing import Any, Callable def total_size(o: Any, handlers: Any = {}) -> int: @@ -15,8 +15,7 @@ def total_size(o: Any, handlers: Any = {}) -> int: Based on https://github.com/ActiveState/recipe-577504-compute-mem-footprint/blob/master/recipe.py """ - def dict_handler(d: Dict) -> chain[Any]: - return chain.from_iterable(d.items()) + dict_handler: Callable[[Any], chain[Any]] = lambda d: chain.from_iterable(d.items()) all_handlers = { tuple: iter, diff --git a/metadata-ingestion/tests/unit/utilities/test_memory_footprint.py b/metadata-ingestion/tests/unit/utilities/test_memory_footprint.py new file mode 100644 index 00000000000000..703cb55f55686e --- /dev/null +++ b/metadata-ingestion/tests/unit/utilities/test_memory_footprint.py @@ -0,0 +1,27 @@ +from collections import defaultdict + +from datahub.utilities import memory_footprint + + +def test_total_size_with_empty_dict(): + size = memory_footprint.total_size({}) + # Only asserting if it is bigger than 0 because the actual sizes differs per python version + assert size > 0 + + +def test_total_size_with_list(): + size = memory_footprint.total_size({"1": [1, 2, 3, 4]}) + # Only asserting if it is bigger than 0 because the actual sizes differs per python version + assert size > 0 + + +def test_total_size_with_none(): + size = memory_footprint.total_size(None) + # Only asserting if it is bigger than 0 because the actual sizes differs per python version + assert size > 0 + + +def test_total_size_with_defaultdict(): + size = memory_footprint.total_size(defaultdict) + # Only asserting if it is bigger than 0 because the actual sizes differs per python version + assert size > 0 From 65e790ed7eebdca764f18eb8d8536ebb88d8a8c6 Mon Sep 17 00:00:00 2001 From: jx2lee <63435794+jx2lee@users.noreply.github.com> Date: Wed, 19 Oct 2022 10:00:54 +0900 Subject: [PATCH 5/6] docs(quickstart): enable slack community link (#6209) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 이재준 --- docs/quickstart.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/quickstart.md b/docs/quickstart.md index 83efe2fc1f262d..075517f9338ff3 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -118,6 +118,7 @@ docker system prune Still stuck? + Hop over to our [Slack community](https://slack.datahubproject.io) and ask for help in the [#troubleshoot](https://datahubspace.slack.com/archives/C029A3M079U) channel! From 98eba90edc5762123b3e3f96339bc7980b45cca7 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 19 Oct 2022 12:26:22 +0530 Subject: [PATCH 6/6] fix(build): allow image tag via env, fix requirements (#6237) --- .github/workflows/docker-unified.yml | 2 +- docker/docker-compose.consumers-without-neo4j.yml | 4 ++-- docker/docker-compose.consumers.yml | 4 ++-- .../docker-compose.consumers-without-neo4j.quickstart.yml | 4 ++-- docker/quickstart/docker-compose.consumers.quickstart.yml | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index b73b43b9238b96..159fae1f0b470b 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -355,7 +355,7 @@ jobs: smoke_test: name: Run Smoke Tests runs-on: ubuntu-latest - needs: [setup,gms_build,frontend_build,kafka_setup_build,mysql_setup_build,elasticsearch_setup_build] + needs: [setup,gms_build,frontend_build,kafka_setup_build,mysql_setup_build,elasticsearch_setup_build,mae_consumer_build,mce_consumer_build] steps: - name: Check out the repo uses: actions/checkout@v2 diff --git a/docker/docker-compose.consumers-without-neo4j.yml b/docker/docker-compose.consumers-without-neo4j.yml index 0a6c6b7089161f..6830916e442042 100644 --- a/docker/docker-compose.consumers-without-neo4j.yml +++ b/docker/docker-compose.consumers-without-neo4j.yml @@ -10,7 +10,7 @@ services: build: context: ../ dockerfile: docker/datahub-mae-consumer/Dockerfile - image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-head} + image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head} env_file: datahub-mae-consumer/env/docker-without-neo4j.env hostname: datahub-mae-consumer container_name: datahub-mae-consumer @@ -24,7 +24,7 @@ services: build: context: ../ dockerfile: docker/datahub-mce-consumer/Dockerfile - image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-head} + image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head} env_file: datahub-mce-consumer/env/docker.env hostname: datahub-mce-consumer container_name: datahub-mce-consumer diff --git a/docker/docker-compose.consumers.yml b/docker/docker-compose.consumers.yml index 3eb8580a61a42f..0eac8ea920b51d 100644 --- a/docker/docker-compose.consumers.yml +++ b/docker/docker-compose.consumers.yml @@ -10,7 +10,7 @@ services: build: context: ../ dockerfile: docker/datahub-mae-consumer/Dockerfile - image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-head} + image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head} env_file: datahub-mae-consumer/env/docker.env hostname: datahub-mae-consumer container_name: datahub-mae-consumer @@ -25,7 +25,7 @@ services: build: context: ../ dockerfile: docker/datahub-mce-consumer/Dockerfile - image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-head} + image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head} env_file: datahub-mce-consumer/env/docker.env hostname: datahub-mce-consumer container_name: datahub-mce-consumer diff --git a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml index 966e3542074814..97180d61989d08 100644 --- a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml @@ -20,7 +20,7 @@ services: - GRAPH_SERVICE_IMPL=elasticsearch - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml hostname: datahub-mae-consumer - image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-head} + image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head} ports: - 9091:9091 datahub-mce-consumer: @@ -35,7 +35,7 @@ services: - DATAHUB_GMS_HOST=datahub-gms - DATAHUB_GMS_PORT=8080 hostname: datahub-mce-consumer - image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-head} + image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head} ports: - 9090:9090 version: '2.3' diff --git a/docker/quickstart/docker-compose.consumers.quickstart.yml b/docker/quickstart/docker-compose.consumers.quickstart.yml index 700f48d42de272..a24d58a11207bb 100644 --- a/docker/quickstart/docker-compose.consumers.quickstart.yml +++ b/docker/quickstart/docker-compose.consumers.quickstart.yml @@ -25,7 +25,7 @@ services: - GRAPH_SERVICE_IMPL=neo4j - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml hostname: datahub-mae-consumer - image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-head} + image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head} ports: - 9091:9091 datahub-mce-consumer: @@ -40,7 +40,7 @@ services: - DATAHUB_GMS_HOST=datahub-gms - DATAHUB_GMS_PORT=8080 hostname: datahub-mce-consumer - image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-head} + image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head} ports: - 9090:9090 version: '2.3'