Skip to content

Commit

Permalink
Merge branch 'master' into jj--browse-paths-feast-sagemaker
Browse files Browse the repository at this point in the history
  • Loading branch information
jjoyce0510 authored Dec 5, 2022
2 parents 614b446 + e5a823e commit a7d07bb
Show file tree
Hide file tree
Showing 109 changed files with 2,173 additions and 330 deletions.
9 changes: 9 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,11 @@ Here are the companies that have officially adopted DataHub. Please feel free to
- [Adevinta](https://www.adevinta.com/)
- [Banksalad](https://www.banksalad.com)
- [Cabify](https://cabify.tech/)
- [ClassDojo](https://www.classdojo.com/)
- [Coursera](https://www.coursera.org/)
- [DefinedCrowd](http://www.definedcrowd.com)
- [DFDS](https://www.dfds.com/)
- [Digital Turbine](https://www.digitalturbine.com/)
- [Expedia Group](http://expedia.com)
- [Experius](https://www.experius.nl)
- [Geotab](https://www.geotab.com)
Expand All @@ -122,15 +125,21 @@ Here are the companies that have officially adopted DataHub. Please feel free to
- [Klarna](https://www.klarna.com)
- [LinkedIn](http://linkedin.com)
- [Moloco](https://www.moloco.com/en)
- [N26](https://n26brasil.com/)
- [Optum](https://www.optum.com/)
- [Peloton](https://www.onepeloton.com)
- [Razer](https://www.razer.com)
- [Saxo Bank](https://www.home.saxo)
- [Showroomprive](https://www.showroomprive.com/)
- [SpotHero](https://spothero.com)
- [Stash](https://www.stash.com)
- [Shanghai HuaRui Bank](https://www.shrbank.com)
- [ThoughtWorks](https://www.thoughtworks.com)
- [TypeForm](http://typeform.com)
- [Udemy](https://www.udemy.com/)
- [Uphold](https://uphold.com)
- [Viasat](https://viasat.com)
- [Wikimedia](https://www.wikimedia.org)
- [Wolt](https://wolt.com)
- [Zynga](https://www.zynga.com)

Expand Down
17 changes: 11 additions & 6 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ buildscript {
ext.testContainersVersion = '1.17.4'
ext.jacksonVersion = '2.13.4'
ext.jettyVersion = '9.4.46.v20220331'
ext.log4jVersion = '2.19.0'
ext.slf4jVersion = '1.7.32'
ext.logbackClassic = '1.2.11'

apply from: './repositories.gradle'
buildscript.repositories.addAll(project.repositories)
dependencies {
Expand Down Expand Up @@ -55,6 +59,8 @@ project.ext.externalDependency = [
'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.10',
'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.1',
'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.8',
'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.0',
'awsRds':'software.amazon.awssdk:rds:2.18.24',
'cacheApi' : 'javax.cache:cache-api:1.1.0',
'commonsCli': 'commons-cli:commons-cli:1.5.0',
'commonsIo': 'commons-io:commons-io:2.4',
Expand Down Expand Up @@ -112,9 +118,11 @@ project.ext.externalDependency = [
'kafkaAvroSerde': 'io.confluent:kafka-streams-avro-serde:5.5.1',
'kafkaAvroSerializer': 'io.confluent:kafka-avro-serializer:5.1.4',
'kafkaClients': 'org.apache.kafka:kafka-clients:2.3.0',
'logbackClassic': 'ch.qos.logback:logback-classic:1.2.9',
'log4jCore': 'org.apache.logging.log4j:log4j-core:2.19.0',
'log4jApi': 'org.apache.logging.log4j:log4j-api:2.19.0',
'logbackClassic': "ch.qos.logback:logback-classic:$logbackClassic",
'slf4jApi': "org.slf4j:slf4j-api:$slf4jVersion",
'log4jCore': "org.apache.logging.log4j:log4j-core:$log4jVersion",
'log4jApi': "org.apache.logging.log4j:log4j-api:$log4jVersion",
'log4j12Api': "org.slf4j:log4j-over-slf4j:$slf4jVersion",
'lombok': 'org.projectlombok:lombok:1.18.12',
'mariadbConnector': 'org.mariadb.jdbc:mariadb-java-client:2.6.0',
'mavenArtifact': "org.apache.maven:maven-artifact:$mavenVersion",
Expand Down Expand Up @@ -193,15 +201,12 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage') }) {
exclude group: "io.netty", module: "netty"
exclude group: "log4j", module: "log4j"
exclude group: "org.springframework.boot", module: "spring-boot-starter-logging"
exclude group: "ch.qos.logback", module: "logback-classic"
exclude group: "org.apache.logging.log4j", module: "log4j-to-slf4j"
exclude group: "com.vaadin.external.google", module: "android-json"
exclude group: "org.slf4j", module: "slf4j-reload4j"
exclude group: "org.slf4j", module: "slf4j-log4j12"
exclude group: "org.slf4j", module: "slf4j-nop"
exclude group: "org.slf4j", module: "slf4j-ext"
exclude group: "org.slf4j", module: "jul-to-slf4j"
exclude group: "org.slf4j", module: "jcl-over-toslf4j"
}
}

Expand Down
3 changes: 2 additions & 1 deletion datahub-frontend/play.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,14 @@ dependencies {
testImplementation externalDependency.playTest
testCompile externalDependency.testng

implementation externalDependency.slf4jApi
compileOnly externalDependency.lombok
runtime externalDependency.guice
runtime (externalDependency.playDocs) {
exclude group: 'com.typesafe.akka', module: 'akka-http-core_2.12'
}
runtime externalDependency.playGuice
runtime externalDependency.logbackClassic
implementation externalDependency.logbackClassic

annotationProcessor externalDependency.lombok
}
Expand Down
1 change: 1 addition & 0 deletions datahub-graphql-core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ dependencies {
compile externalDependency.antlr4
compile externalDependency.guava

implementation externalDependency.slf4jApi
compileOnly externalDependency.lombok
annotationProcessor externalDependency.lombok

Expand Down
2 changes: 1 addition & 1 deletion datahub-ranger-plugin/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ dependencies {
exclude group: "org.apache.htrace", module: "htrace-core4"
}
implementation externalDependency.hadoopCommon3
implementation externalDependency.log4jApi
implementation externalDependency.log4j12Api

constraints {
implementation(externalDependency.woodstoxCore) {
Expand Down
3 changes: 2 additions & 1 deletion datahub-upgrade/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ dependencies {
exclude group: 'com.nimbusds', module: 'nimbus-jose-jwt'
exclude group: "org.apache.htrace", module: "htrace-core4"
}
compile externalDependency.lombok
implementation externalDependency.slf4jApi
compileOnly externalDependency.lombok
compile externalDependency.picocli
compile externalDependency.parquet
compile externalDependency.springBeans
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { Message } from '../../shared/Message';
import { useListDomainsQuery } from '../../../graphql/domain.generated';
import filterSearchQuery from '../../search/utils/filterSearchQuery';
import { ANTD_GRAY } from '../../entity/shared/constants';
import { useGetAuthenticatedUser } from '../../useGetAuthenticatedUser';

const HighlightGroup = styled.div`
display: flex;
Expand Down Expand Up @@ -46,13 +47,16 @@ const StyledSearchBar = styled(Input)`
`;

export const AnalyticsPage = () => {
const me = useGetAuthenticatedUser();
const canManageDomains = me?.platformPrivileges?.createDomains;
const { data: chartData, loading: chartLoading, error: chartError } = useGetAnalyticsChartsQuery();
const { data: highlightData, loading: highlightLoading, error: highlightError } = useGetHighlightsQuery();
const {
loading: domainLoading,
error: domainError,
data: domainData,
} = useListDomainsQuery({
skip: !canManageDomains,
variables: {
input: {
start: 0,
Expand Down Expand Up @@ -82,12 +86,11 @@ export const AnalyticsPage = () => {
skip: domain === '' && query === '',
});

const isLoading = highlightLoading || chartLoading || domainLoading || metadataAnalyticsLoading;
return (
<>
{isLoading && <Message type="loading" content="Loading..." style={{ marginTop: '10%' }} />}
<HighlightGroup>
{highlightLoading && (
<Message type="loading" content="Loading Highlights..." style={{ marginTop: '10%' }} />
)}
{highlightError && (
<Alert type="error" message={highlightError?.message || 'Highlights failed to load'} />
)}
Expand All @@ -96,7 +99,6 @@ export const AnalyticsPage = () => {
))}
</HighlightGroup>
<>
{chartLoading && <Message type="loading" content="Loading Charts..." style={{ marginTop: '10%' }} />}
{chartError && (
<Alert type="error" message={metadataAnalyticsError?.message || 'Charts failed to load'} />
)}
Expand All @@ -107,7 +109,6 @@ export const AnalyticsPage = () => {
))}
</>
<>
{domainLoading && <Message type="loading" content="Loading Domains..." style={{ marginTop: '10%' }} />}
{domainError && (
<Alert type="error" message={metadataAnalyticsError?.message || 'Domains failed to load'} />
)}
Expand Down Expand Up @@ -148,9 +149,6 @@ export const AnalyticsPage = () => {
)}
</>
<>
{metadataAnalyticsLoading && (
<Message type="loading" content="Loading Charts..." style={{ marginTop: '10%' }} />
)}
{metadataAnalyticsError && (
<Alert type="error" message={metadataAnalyticsError?.message || 'Charts failed to load'} />
)}
Expand All @@ -165,7 +163,6 @@ export const AnalyticsPage = () => {
))}
</>
<>
{chartLoading && <Message type="loading" content="Loading Charts..." style={{ marginTop: '10%' }} />}
{chartError && <Alert type="error" message={chartError?.message || 'Charts failed to load'} />}
{!chartLoading &&
chartData?.getAnalyticsCharts
Expand Down
Original file line number Diff line number Diff line change
@@ -1,163 +1,23 @@
import React, { useState } from 'react';
import { Table, Typography } from 'antd';
import { CheckSquareOutlined } from '@ant-design/icons';
import { AlignType } from 'rc-table/lib/interface';
import styled from 'styled-components';
import { Link } from 'react-router-dom';
import React from 'react';

import MlFeatureDataTypeIcon from './MlFeatureDataTypeIcon';
import { MlFeatureDataType, MlPrimaryKey, MlFeature } from '../../../../../types.generated';
import { MlPrimaryKey, MlFeature } from '../../../../../types.generated';
import { GetMlFeatureTableQuery } from '../../../../../graphql/mlFeatureTable.generated';
import { useBaseEntity, useRefetch } from '../../../shared/EntityContext';
import { useBaseEntity } from '../../../shared/EntityContext';
import { notEmpty } from '../../../shared/utils';
import TagTermGroup from '../../../../shared/tags/TagTermGroup';
import SchemaDescriptionField from '../../../dataset/profile/schema/components/SchemaDescriptionField';
import { useUpdateDescriptionMutation } from '../../../../../graphql/mutations.generated';
import { useEntityRegistry } from '../../../../useEntityRegistry';

const FeaturesContainer = styled.div`
margin-bottom: 100px;
`;

const defaultColumns = [
{
title: 'Type',
dataIndex: 'dataType',
key: 'dataType',
width: 100,
align: 'left' as AlignType,
render: (dataType: MlFeatureDataType) => {
return <MlFeatureDataTypeIcon dataType={dataType} />;
},
},
];
import TableOfMlFeatures from './TableOfMlFeatures';

export default function MlFeatureTableFeatures() {
const baseEntity = useBaseEntity<GetMlFeatureTableQuery>();
const refetch = useRefetch();
const featureTable = baseEntity?.mlFeatureTable;
const [updateDescription] = useUpdateDescriptionMutation();
const entityRegistry = useEntityRegistry();

const [tagHoveredIndex, setTagHoveredIndex] = useState<string | undefined>(undefined);

const features =
const features = (
featureTable?.properties && (featureTable?.properties?.mlFeatures || featureTable?.properties?.mlPrimaryKeys)
? [
...(featureTable?.properties?.mlPrimaryKeys || []),
...(featureTable?.properties?.mlFeatures || []),
].filter(notEmpty)
: [];

const onTagTermCell = (record: any, rowIndex: number | undefined) => ({
onMouseEnter: () => {
setTagHoveredIndex(`${record.urn}-${rowIndex}`);
},
onMouseLeave: () => {
setTagHoveredIndex(undefined);
},
});

const nameColumn = {
title: 'Name',
dataIndex: 'name',
key: 'name',
width: 100,
render: (name: string, feature: MlFeature | MlPrimaryKey) => (
<Link to={entityRegistry.getEntityUrl(feature.type, feature.urn)}>
<Typography.Text strong>{name}</Typography.Text>
</Link>
),
};

const descriptionColumn = {
title: 'Description',
dataIndex: 'description',
key: 'description',
render: (_, feature: MlFeature | MlPrimaryKey) => (
<SchemaDescriptionField
description={feature?.editableProperties?.description || feature?.properties?.description || ''}
original={feature?.properties?.description}
isEdited={!!feature?.editableProperties?.description}
onUpdate={(updatedDescription) =>
updateDescription({
variables: {
input: {
description: updatedDescription,
resourceUrn: feature.urn,
},
},
}).then(refetch)
}
/>
),
width: 300,
};

const tagColumn = {
width: 125,
title: 'Tags',
dataIndex: 'tags',
key: 'tags',
render: (_, feature: MlFeature | MlPrimaryKey, rowIndex: number) => (
<TagTermGroup
editableTags={feature.tags}
canRemove
buttonProps={{ size: 'small' }}
canAddTag={tagHoveredIndex === `${feature.urn}-${rowIndex}`}
onOpenModal={() => setTagHoveredIndex(undefined)}
entityUrn={feature.urn}
entityType={feature.type}
refetch={refetch}
/>
),
onCell: onTagTermCell,
};

const termColumn = {
width: 125,
title: 'Terms',
dataIndex: 'glossaryTerms',
key: 'glossarTerms',
render: (_, feature: MlFeature | MlPrimaryKey, rowIndex: number) => (
<TagTermGroup
editableGlossaryTerms={feature.glossaryTerms}
canRemove
buttonProps={{ size: 'small' }}
canAddTerm={tagHoveredIndex === `${feature.urn}-${rowIndex}`}
onOpenModal={() => setTagHoveredIndex(undefined)}
entityUrn={feature.urn}
entityType={feature.type}
refetch={refetch}
/>
),
onCell: onTagTermCell,
};

const primaryKeyColumn = {
title: 'Primary Key',
dataIndex: 'primaryKey',
key: 'primaryKey',
render: (_: any, record: MlFeature | MlPrimaryKey) =>
record.__typename === 'MLPrimaryKey' ? <CheckSquareOutlined /> : null,
width: 50,
};

const allColumns = [...defaultColumns, nameColumn, descriptionColumn, tagColumn, termColumn, primaryKeyColumn];
: []
) as Array<MlFeature | MlPrimaryKey>;

return (
<FeaturesContainer>
{features && features.length > 0 && (
<Table
columns={allColumns}
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
dataSource={features}
rowKey={(record) => `${record.dataType}-${record.name}`}
expandable={{ defaultExpandAllRows: true, expandRowByClick: true }}
pagination={false}
/>
)}
</FeaturesContainer>
);
return <TableOfMlFeatures features={features} />;
}
Loading

0 comments on commit a7d07bb

Please sign in to comment.