Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(ui): Supporting display of columns and storage count in previews #7198

Merged
merged 9 commits into from
Feb 2, 2023
Merged

feat(ui): Supporting display of columns and storage count in previews #7198

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
7eadbc1
Supporting display of columsn and storage count in previews
jjoyce0510 Jan 31, 2023
edde79d
Adding some tests
jjoyce0510 Jan 31, 2023
0679108
Adding a new test
jjoyce0510 Jan 31, 2023
7a2219a
Fix lint
jjoyce0510 Jan 31, 2023
f046a39
Adding a bunch more stuff as well
jjoyce0510 Jan 31, 2023
5dba3b8
Adding formatted bytes stat
jjoyce0510 Feb 1, 2023
a304f71
Mocks updates
jjoyce0510 Feb 1, 2023
92b911d
Merge branch 'master' into jj--add-storage-size-column-count-to-stats
jjoyce0510 Feb 1, 2023
a9217ed
Merge branch 'master' into jj--add-storage-size-column-count-to-stats
jjoyce0510 Feb 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ...rc/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetProfileMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ public com.linkedin.datahub.graphql.generated.DatasetProfile apply(@Nonnull fina

result.setRowCount(gmsProfile.getRowCount());
result.setColumnCount(gmsProfile.getColumnCount());
result.setSizeInBytes(gmsProfile.getSizeInBytes());
result.setTimestampMillis(gmsProfile.getTimestampMillis());
if (gmsProfile.hasFieldProfiles()) {
result.setFieldProfiles(
Expand Down
5 changes: 5 additions & 0 deletions datahub-graphql-core/src/main/resources/entity.graphql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6039,6 +6039,11 @@ type DatasetProfile implements TimeSeriesAspect {
"""
columnCount: Long

"""
The storage size in bytes
"""
sizeInBytes: Long

"""
An optional set of per field statistics obtained in the profile
"""
Expand Down
187 changes: 187 additions & 0 deletions ...est/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetProfileMapperTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
package com.linkedin.datahub.graphql.types.dataset.mappers;

import com.google.common.collect.ImmutableList;
import com.linkedin.data.template.StringArray;
import com.linkedin.datahub.graphql.generated.DatasetProfile;
import com.linkedin.dataset.DatasetFieldProfile;
import com.linkedin.dataset.DatasetFieldProfileArray;
import com.linkedin.metadata.aspect.EnvelopedAspect;
import com.linkedin.metadata.utils.GenericRecordUtils;
import java.util.ArrayList;
import org.testng.Assert;
import org.testng.annotations.Test;

public class DatasetProfileMapperTest {
@Test
public void testMapperFullProfile() {
final com.linkedin.dataset.DatasetProfile input = new com.linkedin.dataset.DatasetProfile();
input.setTimestampMillis(1L);
input.setRowCount(10L);
input.setColumnCount(45L);
input.setSizeInBytes(15L);
input.setFieldProfiles(new DatasetFieldProfileArray(ImmutableList.of(
new DatasetFieldProfile().setFieldPath("/field1")
.setMax("1")
.setMean("2")
.setStdev("3")
.setMedian("4")
.setMin("5")
.setNullCount(20L)
.setNullProportion(20.5f)
.setUniqueCount(30L)
.setUniqueProportion(30.5f)
.setSampleValues(new StringArray(ImmutableList.of("val1", "val2"))),
new DatasetFieldProfile().setFieldPath("/field2")
.setMax("2")
.setMean("3")
.setStdev("4")
.setMedian("5")
.setMin("6")
.setNullCount(30L)
.setNullProportion(30.5f)
.setUniqueCount(40L)
.setUniqueProportion(40.5f)
.setSampleValues(new StringArray(ImmutableList.of("val3", "val4")))
)));
final EnvelopedAspect inputAspect = new EnvelopedAspect()
.setAspect(GenericRecordUtils.serializeAspect(input));
final DatasetProfile actual = DatasetProfileMapper.map(inputAspect);
final DatasetProfile expected = new DatasetProfile();
expected.setTimestampMillis(1L);
expected.setRowCount(10L);
expected.setColumnCount(45L);
expected.setSizeInBytes(15L);
expected.setFieldProfiles(new ArrayList<>(
ImmutableList.of(
new com.linkedin.datahub.graphql.generated.DatasetFieldProfile("/field1",
30L,
30.5f,
20L,
20.5f,
"5",
"1",
"2",
"4",
"3",
new ArrayList<>(ImmutableList.of("val1", "val2"))),
new com.linkedin.datahub.graphql.generated.DatasetFieldProfile("/field2",
40L,
40.5f,
30L,
30.5f,
"6",
"2",
"3",
"5",
"4",
new ArrayList<>(ImmutableList.of("val3", "val4")))
)
));
Assert.assertEquals(actual.getTimestampMillis(), expected.getTimestampMillis());
Assert.assertEquals(actual.getRowCount(), expected.getRowCount());
Assert.assertEquals(actual.getColumnCount(), expected.getColumnCount());
Assert.assertEquals(actual.getSizeInBytes(), expected.getSizeInBytes());

Assert.assertEquals(actual.getFieldProfiles().get(0).getFieldPath(), expected.getFieldProfiles().get(0).getFieldPath());
Assert.assertEquals(actual.getFieldProfiles().get(0).getMax(), expected.getFieldProfiles().get(0).getMax());
Assert.assertEquals(actual.getFieldProfiles().get(0).getMean(), expected.getFieldProfiles().get(0).getMean());
Assert.assertEquals(actual.getFieldProfiles().get(0).getMedian(), expected.getFieldProfiles().get(0).getMedian());
Assert.assertEquals(actual.getFieldProfiles().get(0).getNullCount(), expected.getFieldProfiles().get(0).getNullCount());
Assert.assertEquals(actual.getFieldProfiles().get(0).getNullProportion(), expected.getFieldProfiles().get(0).getNullProportion());
Assert.assertEquals(actual.getFieldProfiles().get(0).getStdev(), expected.getFieldProfiles().get(0).getStdev());
Assert.assertEquals(actual.getFieldProfiles().get(0).getUniqueCount(), expected.getFieldProfiles().get(0).getUniqueCount());
Assert.assertEquals(actual.getFieldProfiles().get(0).getUniqueProportion(), expected.getFieldProfiles().get(0).getUniqueProportion());
Assert.assertEquals(actual.getFieldProfiles().get(0).getSampleValues(), expected.getFieldProfiles().get(0).getSampleValues());

Assert.assertEquals(actual.getFieldProfiles().get(1).getFieldPath(), expected.getFieldProfiles().get(1).getFieldPath());
Assert.assertEquals(actual.getFieldProfiles().get(1).getMax(), expected.getFieldProfiles().get(1).getMax());
Assert.assertEquals(actual.getFieldProfiles().get(1).getMean(), expected.getFieldProfiles().get(1).getMean());
Assert.assertEquals(actual.getFieldProfiles().get(1).getMedian(), expected.getFieldProfiles().get(1).getMedian());
Assert.assertEquals(actual.getFieldProfiles().get(1).getNullCount(), expected.getFieldProfiles().get(1).getNullCount());
Assert.assertEquals(actual.getFieldProfiles().get(1).getNullProportion(), expected.getFieldProfiles().get(1).getNullProportion());
Assert.assertEquals(actual.getFieldProfiles().get(1).getStdev(), expected.getFieldProfiles().get(1).getStdev());
Assert.assertEquals(actual.getFieldProfiles().get(1).getUniqueCount(), expected.getFieldProfiles().get(1).getUniqueCount());
Assert.assertEquals(actual.getFieldProfiles().get(1).getUniqueProportion(), expected.getFieldProfiles().get(1).getUniqueProportion());
Assert.assertEquals(actual.getFieldProfiles().get(1).getSampleValues(), expected.getFieldProfiles().get(1).getSampleValues());
}

@Test
public void testMapperPartialProfile() {
final com.linkedin.dataset.DatasetProfile input = new com.linkedin.dataset.DatasetProfile();
input.setTimestampMillis(1L);
input.setRowCount(10L);
input.setColumnCount(45L);
input.setFieldProfiles(new DatasetFieldProfileArray(ImmutableList.of(
new DatasetFieldProfile().setFieldPath("/field1")
.setUniqueCount(30L)
.setUniqueProportion(30.5f),
new DatasetFieldProfile().setFieldPath("/field2")
.setMax("2")
.setMean("3")
.setStdev("4")
.setMedian("5")
.setMin("6")
.setUniqueCount(40L)
.setUniqueProportion(40.5f)
)));
final EnvelopedAspect inputAspect = new EnvelopedAspect()
.setAspect(GenericRecordUtils.serializeAspect(input));
final DatasetProfile actual = DatasetProfileMapper.map(inputAspect);
final DatasetProfile expected = new DatasetProfile();
expected.setTimestampMillis(1L);
expected.setRowCount(10L);
expected.setColumnCount(45L);
expected.setFieldProfiles(new ArrayList<>(
ImmutableList.of(
new com.linkedin.datahub.graphql.generated.DatasetFieldProfile("/field1",
30L,
30.5f,
null,
null,
null,
null,
null,
null,
null,
null),
new com.linkedin.datahub.graphql.generated.DatasetFieldProfile("/field2",
40L,
40.5f,
null,
null,
"6",
"2",
"3",
"5",
"4",
null)
)
));
Assert.assertEquals(actual.getTimestampMillis(), expected.getTimestampMillis());
Assert.assertEquals(actual.getRowCount(), expected.getRowCount());
Assert.assertEquals(actual.getColumnCount(), expected.getColumnCount());
Assert.assertEquals(actual.getSizeInBytes(), expected.getSizeInBytes());

Assert.assertEquals(actual.getFieldProfiles().get(0).getFieldPath(), expected.getFieldProfiles().get(0).getFieldPath());
Assert.assertEquals(actual.getFieldProfiles().get(0).getMax(), expected.getFieldProfiles().get(0).getMax());
Assert.assertEquals(actual.getFieldProfiles().get(0).getMean(), expected.getFieldProfiles().get(0).getMean());
Assert.assertEquals(actual.getFieldProfiles().get(0).getMedian(), expected.getFieldProfiles().get(0).getMedian());
Assert.assertEquals(actual.getFieldProfiles().get(0).getNullCount(), expected.getFieldProfiles().get(0).getNullCount());
Assert.assertEquals(actual.getFieldProfiles().get(0).getNullProportion(), expected.getFieldProfiles().get(0).getNullProportion());
Assert.assertEquals(actual.getFieldProfiles().get(0).getStdev(), expected.getFieldProfiles().get(0).getStdev());
Assert.assertEquals(actual.getFieldProfiles().get(0).getUniqueCount(), expected.getFieldProfiles().get(0).getUniqueCount());
Assert.assertEquals(actual.getFieldProfiles().get(0).getUniqueProportion(), expected.getFieldProfiles().get(0).getUniqueProportion());
Assert.assertEquals(actual.getFieldProfiles().get(0).getSampleValues(), expected.getFieldProfiles().get(0).getSampleValues());

Assert.assertEquals(actual.getFieldProfiles().get(1).getFieldPath(), expected.getFieldProfiles().get(1).getFieldPath());
Assert.assertEquals(actual.getFieldProfiles().get(1).getMax(), expected.getFieldProfiles().get(1).getMax());
Assert.assertEquals(actual.getFieldProfiles().get(1).getMean(), expected.getFieldProfiles().get(1).getMean());
Assert.assertEquals(actual.getFieldProfiles().get(1).getMedian(), expected.getFieldProfiles().get(1).getMedian());
Assert.assertEquals(actual.getFieldProfiles().get(1).getNullCount(), expected.getFieldProfiles().get(1).getNullCount());
Assert.assertEquals(actual.getFieldProfiles().get(1).getNullProportion(), expected.getFieldProfiles().get(1).getNullProportion());
Assert.assertEquals(actual.getFieldProfiles().get(1).getStdev(), expected.getFieldProfiles().get(1).getStdev());
Assert.assertEquals(actual.getFieldProfiles().get(1).getUniqueCount(), expected.getFieldProfiles().get(1).getUniqueCount());
Assert.assertEquals(actual.getFieldProfiles().get(1).getUniqueProportion(), expected.getFieldProfiles().get(1).getUniqueProportion());
Assert.assertEquals(actual.getFieldProfiles().get(1).getSampleValues(), expected.getFieldProfiles().get(1).getSampleValues());
}
}
2 changes: 2 additions & 0 deletions datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,8 @@ export class DatasetEntity implements Entity<Dataset> {
externalUrl={data.properties?.externalUrl}
statsSummary={data.statsSummary}
rowCount={(data as any).lastProfile?.length && (data as any).lastProfile[0].rowCount}
columnCount={(data as any).lastProfile?.length && (data as any).lastProfile[0].columnCount}
sizeInBytes={(data as any).lastProfile?.length && (data as any).lastProfile[0].sizeInBytes}
jjoyce0510 marked this conversation as resolved.
Show resolved Hide resolved
lastUpdatedMs={
(data as any).lastOperation?.length && (data as any).lastOperation[0].lastUpdatedTimestamp
}
Expand Down
6 changes: 6 additions & 0 deletions datahub-web-react/src/app/entity/dataset/preview/Preview.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ export const Preview = ({
container,
parentContainers,
rowCount,
columnCount,
sizeInBytes,
statsSummary,
lastUpdatedMs,
}: {
Expand All @@ -65,6 +67,8 @@ export const Preview = ({
container?: Container | null;
parentContainers?: ParentContainersResult | null;
rowCount?: number | null;
columnCount?: number | null;
sizeInBytes?: number | null;
statsSummary?: DatasetStatsSummary | null;
lastUpdatedMs?: number | null;
}): JSX.Element => {
Expand Down Expand Up @@ -97,6 +101,8 @@ export const Preview = ({
subHeader={
<DatasetStatsSummaryView
rowCount={rowCount}
columnCount={columnCount}
sizeInBytes={sizeInBytes}
queryCountLast30Days={statsSummary?.queryCountLast30Days}
uniqueUserCountLast30Days={statsSummary?.uniqueUserCountLast30Days}
lastUpdatedMs={lastUpdatedMs}
Expand Down
4 changes: 4 additions & 0 deletions ...hub-web-react/src/app/entity/dataset/profile/stats/stats/DatasetStatsSummarySubHeader.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,17 @@ export const DatasetStatsSummarySubHeader = () => {
((dataset?.operations?.length || 0) > 0 && (dataset?.operations![0] as Operation)) || undefined;

const rowCount = maybeLastProfile?.rowCount;
const columnCount = maybeLastProfile?.columnCount;
const sizeInBytes = maybeLastProfile?.sizeInBytes;
const queryCountLast30Days = maybeStatsSummary?.queryCountLast30Days;
const uniqueUserCountLast30Days = maybeStatsSummary?.uniqueUserCountLast30Days;
const lastUpdatedMs = maybeLastOperation?.lastUpdatedTimestamp;

return (
<DatasetStatsSummary
rowCount={rowCount}
columnCount={columnCount}
sizeInBytes={sizeInBytes}
queryCountLast30Days={queryCountLast30Days}
uniqueUserCountLast30Days={uniqueUserCountLast30Days}
lastUpdatedMs={lastUpdatedMs}
Expand Down
39 changes: 26 additions & 13 deletions datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,13 @@ import {
TableOutlined,
TeamOutlined,
QuestionCircleOutlined,
HddOutlined,
} from '@ant-design/icons';
import { formatNumberWithoutAbbreviation } from '../../../shared/formatNumber';
import { ANTD_GRAY } from '../../shared/constants';
import { toLocalDateTimeString, toRelativeTimeString } from '../../../shared/time/timeUtils';
import { StatsSummary } from '../../shared/components/styled/StatsSummary';
import { FormattedBytesStat } from './FormattedBytesStat';

const StatText = styled.span`
color: ${ANTD_GRAY[8]};
Expand All @@ -24,40 +26,52 @@ const HelpIcon = styled(QuestionCircleOutlined)`

type Props = {
rowCount?: number | null;
columnCount?: number | null;
sizeInBytes?: number | null;
queryCountLast30Days?: number | null;
uniqueUserCountLast30Days?: number | null;
lastUpdatedMs?: number | null;
};

export const DatasetStatsSummary = ({
rowCount,
columnCount,
sizeInBytes,
queryCountLast30Days,
uniqueUserCountLast30Days,
lastUpdatedMs,
}: Props) => {
const statsViews = [
(!!rowCount && (
!!rowCount && (
<StatText>
<TableOutlined style={{ marginRight: 8, color: ANTD_GRAY[7] }} />
<b>{formatNumberWithoutAbbreviation(rowCount)}</b> rows
{!!columnCount && (
<>
, `<b>{formatNumberWithoutAbbreviation(columnCount)}</b> columns
</>
)}
</StatText>
)) ||
undefined,
(!!queryCountLast30Days && (
),
!!sizeInBytes && (
<StatText>
<HddOutlined style={{ marginRight: 8, color: ANTD_GRAY[7] }} />
<FormattedBytesStat bytes={sizeInBytes} />
</StatText>
),
!!queryCountLast30Days && (
<StatText>
<ConsoleSqlOutlined style={{ marginRight: 8, color: ANTD_GRAY[7] }} />
<b>{formatNumberWithoutAbbreviation(queryCountLast30Days)}</b> queries last month
</StatText>
)) ||
undefined,
(!!uniqueUserCountLast30Days && (
),
!!uniqueUserCountLast30Days && (
<StatText>
<TeamOutlined style={{ marginRight: 8, color: ANTD_GRAY[7] }} />
<b>{formatNumberWithoutAbbreviation(uniqueUserCountLast30Days)}</b> unique users
</StatText>
)) ||
undefined,
(!!lastUpdatedMs && (
),
!!lastUpdatedMs && (
<Popover
content={
<div>
Expand All @@ -73,9 +87,8 @@ export const DatasetStatsSummary = ({
Changed {toRelativeTimeString(lastUpdatedMs)}
</StatText>
</Popover>
)) ||
undefined,
].filter((stat) => stat !== undefined);
),
].filter((stat) => stat);

return <>{statsViews.length > 0 && <StatsSummary stats={statsViews} />}</>;
};
33 changes: 33 additions & 0 deletions datahub-web-react/src/app/shared/__tests__/formatNumber.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { formatBytes } from '../formatNumber';

describe('formatBytes', () => {
it('should property format bytes counts', () => {
expect(formatBytes(0)).toStrictEqual({ number: 0, unit: 'Bytes' });
// Whole Numbers
expect(formatBytes(1)).toStrictEqual({ number: 1, unit: 'Bytes' });
expect(formatBytes(10)).toStrictEqual({ number: 10, unit: 'Bytes' });
expect(formatBytes(100)).toStrictEqual({ number: 100, unit: 'Bytes' });
expect(formatBytes(1000)).toStrictEqual({ number: 1, unit: 'KB' });
expect(formatBytes(10000)).toStrictEqual({ number: 10, unit: 'KB' });
expect(formatBytes(100000)).toStrictEqual({ number: 100, unit: 'KB' });
expect(formatBytes(1000000)).toStrictEqual({ number: 1, unit: 'MB' });
expect(formatBytes(10000000)).toStrictEqual({ number: 10, unit: 'MB' });
expect(formatBytes(100000000)).toStrictEqual({ number: 100, unit: 'MB' });
expect(formatBytes(1000000000)).toStrictEqual({ number: 1, unit: 'GB' });
expect(formatBytes(10000000000)).toStrictEqual({ number: 10, unit: 'GB' });
expect(formatBytes(100000000000)).toStrictEqual({ number: 100, unit: 'GB' });
expect(formatBytes(1000000000000)).toStrictEqual({ number: 1, unit: 'TB' });
expect(formatBytes(10000000000000)).toStrictEqual({ number: 10, unit: 'TB' });
expect(formatBytes(100000000000000)).toStrictEqual({ number: 100, unit: 'TB' });
expect(formatBytes(1000000000000000)).toStrictEqual({ number: 1, unit: 'PB' });
// Decimal Numbers
expect(formatBytes(12)).toStrictEqual({ number: 12, unit: 'Bytes' });
expect(formatBytes(1200)).toStrictEqual({ number: 1.2, unit: 'KB' });
expect(formatBytes(1200000)).toStrictEqual({ number: 1.2, unit: 'MB' });
expect(formatBytes(1200000000)).toStrictEqual({ number: 1.2, unit: 'GB' });
expect(formatBytes(1200000000000)).toStrictEqual({ number: 1.2, unit: 'TB' });
expect(formatBytes(1230000000000)).toStrictEqual({ number: 1.23, unit: 'TB' });
expect(formatBytes(1200000000000000)).toStrictEqual({ number: 1.2, unit: 'PB' });
expect(formatBytes(1230000000000000)).toStrictEqual({ number: 1.23, unit: 'PB' });
});
});
Loading