diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx index bd5a357186bb94..9e9e0ede2a1cef 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx @@ -47,9 +47,19 @@ const RawButton = styled(Button)` display: flex; margin-right: 10px; justify-content: left; + align-items: center; } `; +const RawButtonTitleContainer = styled.span` + display: flex; + align-items: center; +`; + +const RawButtonTitle = styled(Typography.Text)` + margin-left: 6px; +`; + const KeyButton = styled(Button)<{ $highlighted: boolean }>` border-radius: 8px 0px 0px 8px; font-weight: ${(props) => (props.$highlighted ? '600' : '400')}; @@ -206,15 +216,15 @@ export default function SchemaHeader({ {hasRaw && ( setShowRaw(!showRaw)}> {showRaw ? ( - <> - - Tabular - + + + Tabular + ) : ( - <> - - Raw - + + + Raw + )} )} diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx index 572f9aafe738c2..5c9a8ca1ef82d8 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx @@ -117,9 +117,11 @@ function EntityDropdown(props: Props) { }; const pageUrl = window.location.href; + const isGlossaryEntity = entityType === EntityType.GlossaryNode || entityType === EntityType.GlossaryTerm; const entityHasChildren = !!entityData?.children?.total; const canManageGlossaryEntity = !!entityData?.privileges?.canManageEntity; const canCreateGlossaryEntity = !!entityData?.privileges?.canManageChildren; + const canDeleteGlossaryEntity = !entityHasChildren && canManageGlossaryEntity; /** * A default path to redirect to if the entity is deleted. @@ -192,7 +194,7 @@ function EntityDropdown(props: Props) { {menuItems.has(EntityMenuItems.DELETE) && ( - {entityData?.schemaMetadata?.fields.map((field) => ( - {downgradeV2FieldPath(field.fieldPath)} - ))} + {entityData?.schemaMetadata?.fields.map((field) => { + const fieldPath = downgradeV2FieldPath(field.fieldPath); + return ( + + {fieldPath} + + ); + })} )} diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx index dc62e5faf213ea..63b7c3d46a2c29 100644 --- a/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx +++ b/datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx @@ -90,7 +90,7 @@ export function TypeColumn(type: string, record: any) { export function LastExecutionColumn(time: any) { const executionDate = time && new Date(time); const localTime = executionDate && `${executionDate.toLocaleDateString()} at ${executionDate.toLocaleTimeString()}`; - return {localTime || 'N/A'}; + return {localTime || 'None'}; } export function ScheduleColumn(schedule: any, record: any) { @@ -114,10 +114,10 @@ export function LastStatusColumn({ status, record, setFocusExecutionUrn }: LastS const color = getExecutionRequestStatusDisplayColor(status); return ( - {Icon && } + {Icon && } setFocusExecutionUrn(record.lastExecUrn)}> - {text || 'N/A'} + {text || 'Pending...'} diff --git a/datahub-web-react/src/app/ingest/source/executions/IngestionExecutionTable.tsx b/datahub-web-react/src/app/ingest/source/executions/IngestionExecutionTable.tsx index 02539d7740968a..8c81cc36ae3f99 100644 --- a/datahub-web-react/src/app/ingest/source/executions/IngestionExecutionTable.tsx +++ b/datahub-web-react/src/app/ingest/source/executions/IngestionExecutionTable.tsx @@ -38,7 +38,7 @@ export default function IngestionExecutionTable({ dataIndex: 'duration', key: 'duration', render: (durationMs: number) => { - const seconds = (durationMs && `${durationMs / 1000}s`) || 'N/A'; + const seconds = (durationMs && `${durationMs / 1000}s`) || 'None'; return seconds; }, }, diff --git a/datahub-web-react/src/app/ingest/source/executions/IngestionExecutionTableColumns.tsx b/datahub-web-react/src/app/ingest/source/executions/IngestionExecutionTableColumns.tsx index 3d9eaececb3cea..5bf96bc703f1f9 100644 --- a/datahub-web-react/src/app/ingest/source/executions/IngestionExecutionTableColumns.tsx +++ b/datahub-web-react/src/app/ingest/source/executions/IngestionExecutionTableColumns.tsx @@ -27,7 +27,7 @@ const StatusButton = styled(Button)` export function TimeColumn(time: string) { const date = time && new Date(time); const localTime = date && `${date.toLocaleDateString()} at ${date.toLocaleTimeString()}`; - return {localTime || 'N/A'}; + return {localTime || 'None'}; } interface StatusColumnProps { @@ -42,10 +42,10 @@ export function StatusColumn({ status, record, setFocusExecutionUrn }: StatusCol const color = getExecutionRequestStatusDisplayColor(status); return ( - {Icon && } + {Icon && } setFocusExecutionUrn(record.urn)}> - {text || 'N/A'} + {text || 'Pending...'} diff --git a/datahub-web-react/src/app/ingest/source/utils.ts b/datahub-web-react/src/app/ingest/source/utils.ts index 5930a04aa04b97..b2f20c93614e2a 100644 --- a/datahub-web-react/src/app/ingest/source/utils.ts +++ b/datahub-web-react/src/app/ingest/source/utils.ts @@ -61,7 +61,7 @@ export const getExecutionRequestStatusIcon = (status: string) => { (status === ROLLED_BACK && WarningOutlined) || (status === ROLLING_BACK && LoadingOutlined) || (status === ROLLBACK_FAILED && CloseCircleOutlined) || - undefined + ClockCircleOutlined ); }; diff --git a/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx b/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx index 44b4d9b562c242..8fbc932abb14cf 100644 --- a/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx +++ b/datahub-web-react/src/app/permissions/roles/ManageRoles.tsx @@ -1,5 +1,5 @@ import React, { useEffect, useMemo, useState } from 'react'; -import { Button, Empty, message, Pagination, Tooltip } from 'antd'; +import { Button, Empty, message, Pagination, Tooltip, Typography } from 'antd'; import styled from 'styled-components'; import * as QueryString from 'query-string'; import { useLocation } from 'react-router'; @@ -16,6 +16,7 @@ import { useBatchAssignRoleMutation } from '../../../graphql/mutations.generated import { CorpUser, DataHubRole, DataHubPolicy } from '../../../types.generated'; import RoleDetailsModal from './RoleDetailsModal'; import analytics, { EventType } from '../../analytics'; +import { ANTD_GRAY } from '../../entity/shared/constants'; const SourceContainer = styled.div``; @@ -33,6 +34,12 @@ const PageContainer = styled.span` width: 100%; `; +const ActionsContainer = styled.div` + width: 100%; + display: flex; + justify-content: right; +`; + const AddUsersButton = styled(Button)` margin-right: 16px; `; @@ -137,7 +144,7 @@ export const ManageRoles = () => { <> onViewRole(record.role)} - style={{ color: record?.editable ? '#000000' : '#8C8C8C' }} + style={{ color: record?.editable ? '#000000' : ANTD_GRAY[8] }} > {record?.name} @@ -158,13 +165,15 @@ export const ManageRoles = () => { render: (_: any, record: any) => { return ( <> - + {(record?.users.length && ( + + )) || No assigned users} ); }, @@ -174,7 +183,7 @@ export const ManageRoles = () => { key: 'actions', render: (_: any, record: any) => { return ( - <> + { @@ -185,7 +194,7 @@ export const ManageRoles = () => { ADD USERS - + ); }, }, diff --git a/datahub-web-react/src/app/search/SearchFiltersSection.tsx b/datahub-web-react/src/app/search/SearchFiltersSection.tsx index a063b37e6feb7b..a2fca0605b4ec3 100644 --- a/datahub-web-react/src/app/search/SearchFiltersSection.tsx +++ b/datahub-web-react/src/app/search/SearchFiltersSection.tsx @@ -31,7 +31,7 @@ const FiltersHeader = styled.div` font-weight: 600; padding-left: 20px; - padding-right: 20px; + padding-right: 4px; padding-bottom: 8px; width: 100%; diff --git a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx index 2b4f6abb831644..53af15e6266a17 100644 --- a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx +++ b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx @@ -10,9 +10,10 @@ import { DownOutlined, } from '@ant-design/icons'; import { Link } from 'react-router-dom'; -import { Button, Dropdown, Menu } from 'antd'; +import { Button, Dropdown, Menu, Tooltip } from 'antd'; import { useAppConfig } from '../../useAppConfig'; import { useGetAuthenticatedUser } from '../../useGetAuthenticatedUser'; +import { ANTD_GRAY } from '../../entity/shared/constants'; const LinkWrapper = styled.span` margin-right: 0px; @@ -34,6 +35,24 @@ const LinksWrapper = styled.div<{ areLinksHidden?: boolean }>` const MenuItem = styled(Menu.Item)` font-size: 12px; font-weight: bold; + max-width: 240px; +`; + +const NavTitleContainer = styled.span` + display: flex; + align-items: center; + justify-content: left; + padding: 2px; +`; + +const NavTitleText = styled.span` + margin-left: 6px; +`; + +const NavTitleDescription = styled.div` + font-size: 12px; + font-weight: normal; + color: ${ANTD_GRAY[7]}; `; interface Props { @@ -60,7 +79,12 @@ export function HeaderLinks(props: Props) { @@ -69,7 +93,12 @@ export function HeaderLinks(props: Props) { @@ -80,13 +109,21 @@ export function HeaderLinks(props: Props) { - Glossary + + + Glossary + + View and modify your data dictionary {showDomains && ( - Domains + + + Domains + + Manage related groups of data assets )} @@ -103,7 +140,9 @@ export function HeaderLinks(props: Props) { diff --git a/docker/datahub-ingestion/base.Dockerfile b/docker/datahub-ingestion/base.Dockerfile index 1f6e5d01fac334..205a76fc291cd5 100644 --- a/docker/datahub-ingestion/base.Dockerfile +++ b/docker/datahub-ingestion/base.Dockerfile @@ -29,6 +29,7 @@ RUN apt-get update && apt-get install -y \ zip \ unzip \ ldap-utils \ + openjdk-11-jre-headless \ && curl -L https://github.com/treff7es/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-${DOCKERIZE_ARCH}-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv \ && python -m pip install --upgrade pip wheel setuptools==57.5.0 \ && curl -Lk -o /root/librdkafka-${LIBRDKAFKA_VERSION}.tar.gz https://github.com/edenhill/librdkafka/archive/v${LIBRDKAFKA_VERSION}.tar.gz \ diff --git a/docs-website/src/pages/_components/Logos/index.js b/docs-website/src/pages/_components/Logos/index.js index 8121fd780eee03..da8484fec7eb72 100644 --- a/docs-website/src/pages/_components/Logos/index.js +++ b/docs-website/src/pages/_components/Logos/index.js @@ -61,6 +61,16 @@ const companiesByIndustry = [ imageUrl: "/img/logos/companies/zynga.png", imageSize: "default", }, + { + name: "Hurb", + imageUrl: "/img/logos/companies/hurb.png", + imageSize: "medium", + }, + { + name: "Razer", + imageUrl: "/img/logos/companies/razer.jpeg", + imageSize: "large", + }, ], }, { diff --git a/docs-website/static/img/logos/companies/hurb.png b/docs-website/static/img/logos/companies/hurb.png new file mode 100644 index 00000000000000..3e55de9d4a0391 Binary files /dev/null and b/docs-website/static/img/logos/companies/hurb.png differ diff --git a/docs-website/static/img/logos/companies/razer.jpeg b/docs-website/static/img/logos/companies/razer.jpeg new file mode 100644 index 00000000000000..d546dd34c42fe5 Binary files /dev/null and b/docs-website/static/img/logos/companies/razer.jpeg differ diff --git a/metadata-ingestion/src/datahub/configuration/common.py b/metadata-ingestion/src/datahub/configuration/common.py index 7ee4df10e04698..e134a5a8495b91 100644 --- a/metadata-ingestion/src/datahub/configuration/common.py +++ b/metadata-ingestion/src/datahub/configuration/common.py @@ -217,3 +217,10 @@ def value(self, string: str) -> List[str]: class VersionedConfig(ConfigModel): version: str = "1" + + +class LineageConfig(ConfigModel): + incremental_lineage: bool = Field( + default=True, + description="When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py index 11f7478c1cb9cc..08000ef130790d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py @@ -92,6 +92,7 @@ GlobalTagsClass, TagAssociationClass, ) +from datahub.specific.dataset import DatasetPatchBuilder from datahub.utilities.hive_schema_to_avro import ( HiveColumnToAvroConverter, get_schema_fields_for_hive_column, @@ -818,12 +819,30 @@ def gen_dataset_workunits( upstream_lineage = None if upstream_lineage is not None: - # Emit the lineage work unit - wu = wrap_aspect_as_workunit( - "dataset", dataset_urn, "upstreamLineage", upstream_lineage - ) - yield wu - self.report.report_workunit(wu) + if self.config.incremental_lineage: + patch_builder: DatasetPatchBuilder = DatasetPatchBuilder( + urn=dataset_urn + ) + for upstream in upstream_lineage.upstreams: + patch_builder.add_upstream_lineage(upstream) + + lineage_workunits = [ + MetadataWorkUnit( + id=f"upstreamLineage-for-{dataset_urn}", + mcp_raw=mcp, + ) + for mcp in patch_builder.build() + ] + else: + lineage_workunits = [ + wrap_aspect_as_workunit( + "dataset", dataset_urn, "upstreamLineage", upstream_lineage + ) + ] + + for wu in lineage_workunits: + yield wu + self.report.report_workunit(wu) dataset_properties = DatasetProperties( name=datahub_dataset_name.get_table_display_name(), diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py index 61a839e70916d2..13a15aafe674e8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py @@ -4,7 +4,7 @@ from pydantic import Field, PositiveInt, root_validator -from datahub.configuration.common import AllowDenyPattern +from datahub.configuration.common import AllowDenyPattern, LineageConfig from datahub.ingestion.source.usage.usage_common import BaseUsageConfig from datahub.ingestion.source_config.sql.bigquery import BigQueryConfig @@ -23,7 +23,7 @@ class BigQueryUsageConfig(BaseUsageConfig): ) -class BigQueryV2Config(BigQueryConfig): +class BigQueryV2Config(BigQueryConfig, LineageConfig): project_id_pattern: AllowDenyPattern = Field( default=AllowDenyPattern.allow_all(), description="Regex patterns for project_id to filter in ingestion.", diff --git a/smoke-test/smoke.sh b/smoke-test/smoke.sh index 6b0a337485cbca..033fcfd82c7ff0 100755 --- a/smoke-test/smoke.sh +++ b/smoke-test/smoke.sh @@ -27,4 +27,7 @@ DATAHUB_TELEMETRY_ENABLED=false datahub docker quickstart --standalone_consumers (cd ..; ./gradlew :smoke-test:yarnInstall) +export CYPRESS_ADMIN_USERNAME=${ADMIN_USERNAME:-datahub} +export CYPRESS_ADMIN_PASSWORD=${ADMIN_PASSWORD:-datahub} + pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke.xml diff --git a/smoke-test/tests/cypress/cypress/support/commands.js b/smoke-test/tests/cypress/cypress/support/commands.js index 9a36f2cb35bd92..87229b5bab0204 100644 --- a/smoke-test/tests/cypress/cypress/support/commands.js +++ b/smoke-test/tests/cypress/cypress/support/commands.js @@ -15,8 +15,8 @@ Cypress.Commands.add('login', () => { method: 'POST', url: '/logIn', body: { - username: 'datahub', - password: 'datahub', + username: Cypress.env('ADMIN_USERNAME'), + password: Cypress.env('ADMIN_PASSWORD'), }, retryOnStatusCodeFailure: true, });