diff --git a/datahub-graphql-core/build.gradle b/datahub-graphql-core/build.gradle index 2b0109b1f2fae3..97202044c3bc46 100644 --- a/datahub-graphql-core/build.gradle +++ b/datahub-graphql-core/build.gradle @@ -30,6 +30,7 @@ graphqlCodegen { "$projectDir/src/main/resources/recommendation.graphql".toString(), "$projectDir/src/main/resources/ingestion.graphql".toString(), "$projectDir/src/main/resources/auth.graphql".toString(), + "$projectDir/src/main/resources/timeline.graphql".toString(), ] outputDir = new File("$projectDir/src/mainGeneratedGraphQL/java") packageName = "com.linkedin.datahub.graphql.generated" diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java index 724f08c0677346..bba11c75eef4f5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java @@ -15,6 +15,7 @@ public class Constants { public static final String ANALYTICS_SCHEMA_FILE = "analytics.graphql"; public static final String RECOMMENDATIONS_SCHEMA_FILE = "recommendation.graphql"; public static final String INGESTION_SCHEMA_FILE = "ingestion.graphql"; + public static final String TIMELINE_SCHEMA_FILE = "timeline.graphql"; public static final String BROWSE_PATH_DELIMITER = "/"; public static final String VERSION_STAMP_FIELD_NAME = "versionStamp"; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 600dabda56b52f..74106bc5b8c6cc 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -76,10 +76,6 @@ import com.linkedin.datahub.graphql.resolvers.group.ListGroupsResolver; import com.linkedin.datahub.graphql.resolvers.group.RemoveGroupMembersResolver; import com.linkedin.datahub.graphql.resolvers.group.RemoveGroupResolver; -import com.linkedin.datahub.graphql.resolvers.jobs.EntityRunsResolver; -import com.linkedin.datahub.graphql.resolvers.jobs.DataJobRunsResolver; -import com.linkedin.datahub.graphql.resolvers.user.UpdateUserStatusResolver; -import com.linkedin.datahub.graphql.resolvers.policy.GetGrantedPrivilegesResolver; import com.linkedin.datahub.graphql.resolvers.ingest.execution.CancelIngestionExecutionRequestResolver; import com.linkedin.datahub.graphql.resolvers.ingest.execution.CreateIngestionExecutionRequestResolver; import com.linkedin.datahub.graphql.resolvers.ingest.execution.GetIngestionExecutionRequestResolver; @@ -92,6 +88,8 @@ import com.linkedin.datahub.graphql.resolvers.ingest.source.GetIngestionSourceResolver; import com.linkedin.datahub.graphql.resolvers.ingest.source.ListIngestionSourcesResolver; import com.linkedin.datahub.graphql.resolvers.ingest.source.UpsertIngestionSourceResolver; +import com.linkedin.datahub.graphql.resolvers.jobs.DataJobRunsResolver; +import com.linkedin.datahub.graphql.resolvers.jobs.EntityRunsResolver; import com.linkedin.datahub.graphql.resolvers.load.AspectResolver; import com.linkedin.datahub.graphql.resolvers.load.EntityLineageResultResolver; import com.linkedin.datahub.graphql.resolvers.load.EntityRelationshipsResultResolver; @@ -113,6 +111,7 @@ import com.linkedin.datahub.graphql.resolvers.mutate.RemoveTermResolver; import com.linkedin.datahub.graphql.resolvers.mutate.UpdateDescriptionResolver; import com.linkedin.datahub.graphql.resolvers.policy.DeletePolicyResolver; +import com.linkedin.datahub.graphql.resolvers.policy.GetGrantedPrivilegesResolver; import com.linkedin.datahub.graphql.resolvers.policy.ListPoliciesResolver; import com.linkedin.datahub.graphql.resolvers.policy.UpsertPolicyResolver; import com.linkedin.datahub.graphql.resolvers.recommendation.ListRecommendationsResolver; @@ -122,6 +121,7 @@ import com.linkedin.datahub.graphql.resolvers.search.SearchAcrossLineageResolver; import com.linkedin.datahub.graphql.resolvers.search.SearchResolver; import com.linkedin.datahub.graphql.resolvers.tag.SetTagColorResolver; +import com.linkedin.datahub.graphql.resolvers.timeline.ListSchemaBlameResolver; import com.linkedin.datahub.graphql.resolvers.type.AspectInterfaceTypeResolver; import com.linkedin.datahub.graphql.resolvers.type.EntityInterfaceTypeResolver; import com.linkedin.datahub.graphql.resolvers.type.HyperParameterValueTypeResolver; @@ -130,8 +130,8 @@ import com.linkedin.datahub.graphql.resolvers.type.TimeSeriesAspectInterfaceTypeResolver; import com.linkedin.datahub.graphql.resolvers.user.ListUsersResolver; import com.linkedin.datahub.graphql.resolvers.user.RemoveUserResolver; +import com.linkedin.datahub.graphql.resolvers.user.UpdateUserStatusResolver; import com.linkedin.datahub.graphql.types.BrowsableEntityType; -import com.linkedin.datahub.graphql.types.dataprocessinst.mappers.DataProcessInstanceRunEventMapper; import com.linkedin.datahub.graphql.types.EntityType; import com.linkedin.datahub.graphql.types.LoadableType; import com.linkedin.datahub.graphql.types.SearchableEntityType; @@ -144,10 +144,10 @@ import com.linkedin.datahub.graphql.types.corpuser.CorpUserType; import com.linkedin.datahub.graphql.types.dashboard.DashboardType; import com.linkedin.datahub.graphql.types.dataset.VersionedDatasetType; -import com.linkedin.datahub.graphql.types.notebook.NotebookType; import com.linkedin.datahub.graphql.types.dataflow.DataFlowType; import com.linkedin.datahub.graphql.types.datajob.DataJobType; import com.linkedin.datahub.graphql.types.dataplatform.DataPlatformType; +import com.linkedin.datahub.graphql.types.dataprocessinst.mappers.DataProcessInstanceRunEventMapper; import com.linkedin.datahub.graphql.types.dataset.DatasetType; import com.linkedin.datahub.graphql.types.dataset.mappers.DatasetProfileMapper; import com.linkedin.datahub.graphql.types.domain.DomainType; @@ -157,6 +157,7 @@ import com.linkedin.datahub.graphql.types.mlmodel.MLModelGroupType; import com.linkedin.datahub.graphql.types.mlmodel.MLModelType; import com.linkedin.datahub.graphql.types.mlmodel.MLPrimaryKeyType; +import com.linkedin.datahub.graphql.types.notebook.NotebookType; import com.linkedin.datahub.graphql.types.tag.TagType; import com.linkedin.datahub.graphql.types.usage.UsageType; import com.linkedin.entity.client.EntityClient; @@ -166,6 +167,7 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.recommendation.RecommendationsService; import com.linkedin.metadata.secret.SecretService; +import com.linkedin.metadata.timeline.TimelineService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.metadata.version.GitVersion; import com.linkedin.usage.UsageClient; @@ -194,7 +196,7 @@ import static com.linkedin.datahub.graphql.Constants.*; import static com.linkedin.metadata.Constants.*; -import static graphql.Scalars.GraphQLLong; +import static graphql.Scalars.*; /** @@ -216,6 +218,7 @@ public class GmsGraphQLEngine { private final GitVersion gitVersion; private final boolean supportsImpactAnalysis; private final TimeseriesAspectService timeseriesAspectService; + private final TimelineService timelineService; private final IngestionConfiguration ingestionConfiguration; private final AuthenticationConfiguration authenticationConfiguration; @@ -286,6 +289,7 @@ public GmsGraphQLEngine( final AuthenticationConfiguration authenticationConfiguration, final AuthorizationConfiguration authorizationConfiguration, final GitVersion gitVersion, + final TimelineService timelineService, final boolean supportsImpactAnalysis, final VisualConfiguration visualConfiguration ) { @@ -303,6 +307,7 @@ public GmsGraphQLEngine( this.gitVersion = gitVersion; this.supportsImpactAnalysis = supportsImpactAnalysis; this.timeseriesAspectService = timeseriesAspectService; + this.timelineService = timelineService; this.ingestionConfiguration = Objects.requireNonNull(ingestionConfiguration); this.authenticationConfiguration = Objects.requireNonNull(authenticationConfiguration); @@ -417,6 +422,7 @@ public GraphQLEngine.Builder builder() { .addSchema(fileBasedSchema(ANALYTICS_SCHEMA_FILE)) .addSchema(fileBasedSchema(RECOMMENDATIONS_SCHEMA_FILE)) .addSchema(fileBasedSchema(INGESTION_SCHEMA_FILE)) + .addSchema(fileBasedSchema(TIMELINE_SCHEMA_FILE)) .addDataLoaders(loaderSuppliers(loadableTypes)) .addDataLoader("Aspect", context -> createDataLoader(aspectType, context)) .addDataLoader("UsageQueryResult", context -> createDataLoader(usageType, context)) @@ -520,6 +526,7 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("listIngestionSources", new ListIngestionSourcesResolver(this.entityClient)) .dataFetcher("ingestionSource", new GetIngestionSourceResolver(this.entityClient)) .dataFetcher("executionRequest", new GetIngestionExecutionRequestResolver(this.entityClient)) + .dataFetcher("listSchemaBlame", new ListSchemaBlameResolver(this.timelineService)) ); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java index 8fda3c198c482b..bd6731d6d7ff66 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java @@ -13,6 +13,7 @@ import com.linkedin.datahub.graphql.generated.PoliciesConfig; import com.linkedin.datahub.graphql.generated.Privilege; import com.linkedin.datahub.graphql.generated.ResourcePrivileges; +import com.linkedin.datahub.graphql.generated.TimelineConfig; import com.linkedin.datahub.graphql.generated.VisualConfiguration; import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.metadata.version.GitVersion; @@ -90,7 +91,10 @@ public CompletableFuture get(final DataFetchingEnvironment environmen final ManagedIngestionConfig ingestionConfig = new ManagedIngestionConfig(); ingestionConfig.setEnabled(_ingestionConfiguration.isEnabled()); - appConfig.setAuthConfig(authConfig); + + final TimelineConfig timelineConfig = new TimelineConfig(); + timelineConfig.setEnabled(true); + appConfig.setAnalyticsConfig(analyticsConfig); appConfig.setPoliciesConfig(policiesConfig); appConfig.setIdentityManagementConfig(identityManagementConfig); @@ -98,6 +102,8 @@ public CompletableFuture get(final DataFetchingEnvironment environmen appConfig.setAuthConfig(authConfig); appConfig.setVisualConfig(_visualConfiguration); + appConfig.setTimelineConfig(timelineConfig); + return CompletableFuture.completedFuture(appConfig); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/timeline/ListSchemaBlameResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/timeline/ListSchemaBlameResolver.java new file mode 100644 index 00000000000000..d11e0c35501a36 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/timeline/ListSchemaBlameResolver.java @@ -0,0 +1,65 @@ +package com.linkedin.datahub.graphql.resolvers.timeline; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.ListSchemaBlameInput; +import com.linkedin.datahub.graphql.generated.ListSchemaBlameResult; +import com.linkedin.datahub.graphql.types.timeline.mappers.SchemaBlameMapper; +import com.linkedin.metadata.timeline.TimelineService; +import com.linkedin.metadata.timeline.data.ChangeCategory; +import com.linkedin.metadata.timeline.data.ChangeTransaction; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.net.URISyntaxException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +@Slf4j +public class ListSchemaBlameResolver implements DataFetcher> { + private final TimelineService _timelineService; + + private static final Logger _logger = LoggerFactory.getLogger(ListSchemaBlameResolver.class.getName()); + + public ListSchemaBlameResolver(TimelineService timelineService) { + _timelineService = timelineService; + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final ListSchemaBlameInput input = bindArgument(environment.getArgument("input"), ListSchemaBlameInput.class); + + final String datasetUrnString = input.getDatasetUrn() == null ? null : input.getDatasetUrn(); + final long startTime = 0; + final long endTime = 0; + final String versionCutoff = input.getVersionCutoff() == null ? null : input.getVersionCutoff(); + + return CompletableFuture.supplyAsync(() -> { + try { + if (datasetUrnString == null) { + return null; + } + final Set changeCategorySet = new HashSet<>(); + changeCategorySet.add(ChangeCategory.TECHNICAL_SCHEMA); + Urn datasetUrn = Urn.createFromString(datasetUrnString); + List changeTransactionList = + _timelineService.getTimeline(datasetUrn, changeCategorySet, startTime, endTime, null, null, false); + return SchemaBlameMapper.map(changeTransactionList, versionCutoff); + } catch (URISyntaxException u) { + _logger.debug( + String.format("Failed to list schema blame data, likely due to the Urn %s being invalid", datasetUrnString), + u); + return null; + } catch (Exception e) { + _logger.debug("Failed to list schema blame data", e); + return null; + } + }); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/mappers/SchemaBlameMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/mappers/SchemaBlameMapper.java new file mode 100644 index 00000000000000..32b085f9de163d --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/mappers/SchemaBlameMapper.java @@ -0,0 +1,181 @@ +package com.linkedin.datahub.graphql.types.timeline.mappers; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.ChangeOperationType; +import com.linkedin.datahub.graphql.generated.ListSchemaBlameResult; +import com.linkedin.datahub.graphql.generated.SchemaBlame; +import com.linkedin.datahub.graphql.generated.SchemaChange; +import com.linkedin.datahub.graphql.generated.SemVerStruct; +import com.linkedin.datahub.graphql.generated.SemanticVersionChangeType; +import com.linkedin.datahub.graphql.resolvers.timeline.ListSchemaBlameResolver; +import com.linkedin.metadata.key.SchemaFieldKey; +import com.linkedin.metadata.timeline.data.ChangeCategory; +import com.linkedin.metadata.timeline.data.ChangeEvent; +import com.linkedin.metadata.timeline.data.ChangeTransaction; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.util.Pair; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.apache.parquet.SemanticVersion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class SchemaBlameMapper { + + private static final Logger _logger = LoggerFactory.getLogger(ListSchemaBlameResolver.class.getName()); + + public static ListSchemaBlameResult map(List changeTransactions, @Nullable String versionCutoff) { + if (changeTransactions.isEmpty()) { + return null; + } + + Map schemaBlameMap = new HashMap<>(); + ListSchemaBlameResult result = new ListSchemaBlameResult(); + + String latestSemVerString = truncateSemVer(changeTransactions.get(changeTransactions.size() - 1).getSemVer()); + long latestSemVerTimestamp = changeTransactions.get(changeTransactions.size() - 1).getTimestamp(); + String latestVersionStamp = changeTransactions.get(changeTransactions.size() - 1).getVersionStamp(); + result.setLatestSemVer(new SemVerStruct(latestSemVerString, latestSemVerTimestamp, latestVersionStamp)); + + String semVerFilter = versionCutoff == null ? latestSemVerString : versionCutoff; + Optional semanticVersionFilterOptional = createSemanticVersion(semVerFilter); + if (!semanticVersionFilterOptional.isPresent()) { + return result; + } + + SemanticVersion semanticVersionFilter = semanticVersionFilterOptional.get(); + + List reversedChangeTransactions = changeTransactions.stream() + .map(SchemaBlameMapper::semanticVersionChangeTransactionPair) + .filter(Optional::isPresent) + .map(Optional::get) + .filter(semanticVersionChangeTransactionPair -> + semanticVersionChangeTransactionPair.getFirst().compareTo(semanticVersionFilter) <= 0) + .sorted(Collections.reverseOrder(Comparator.comparing(Pair::getFirst))) + .map(Pair::getSecond) + .collect(Collectors.toList()); + + String selectedSemVer = truncateSemVer(reversedChangeTransactions.get(0).getSemVer()); + long selectedSemVerTimestamp = reversedChangeTransactions.get(0).getTimestamp(); + String selectedVersionStamp = reversedChangeTransactions.get(0).getVersionStamp(); + result.setSelectedSemVer(new SemVerStruct(selectedSemVer, selectedSemVerTimestamp, selectedVersionStamp)); + + List semVerList = new ArrayList<>(); + for (ChangeTransaction changeTransaction : reversedChangeTransactions) { + SemVerStruct semVerStruct = + new SemVerStruct(truncateSemVer(changeTransaction.getSemVer()), changeTransaction.getTimestamp(), + changeTransaction.getVersionStamp()); + semVerList.add(semVerStruct); + + for (ChangeEvent changeEvent : changeTransaction.getChangeEvents()) { + if (changeEvent.getCategory() != ChangeCategory.TECHNICAL_SCHEMA) { + continue; + } + + String schemaUrn = changeEvent.getModifier(); + if (schemaUrn == null || schemaBlameMap.containsKey(schemaUrn)) { + continue; + } + + SchemaBlame schemaBlame = new SchemaBlame(); + + SchemaFieldKey schemaFieldKey; + try { + schemaFieldKey = (SchemaFieldKey) EntityKeyUtils.convertUrnToEntityKey(Urn.createFromString(schemaUrn), + new SchemaFieldKey().schema()); + } catch (Exception e) { + _logger.debug(String.format("Could not generate schema urn for %s", schemaUrn)); + continue; + } + + String fieldPath = schemaFieldKey.getFieldPath(); + schemaBlame.setFieldPath(fieldPath); + + SchemaChange schemaChange = + getLastSchemaChange(changeEvent, changeTransaction.getTimestamp(), changeTransaction.getActor(), + changeTransaction.getSemVer(), changeTransaction.getVersionStamp()); + schemaBlame.setSchemaChange(schemaChange); + + schemaBlameMap.put(schemaUrn, schemaBlame); + } + } + + result.setSchemaBlameList(schemaBlameMap.values() + .stream() + .filter(SchemaBlame -> !SchemaBlame.getSchemaChange().getChangeType().equals(ChangeOperationType.REMOVE)) + .collect(Collectors.toList())); + result.setSemVerList(semVerList); + return result; + } + + private static Optional> semanticVersionChangeTransactionPair( + ChangeTransaction changeTransaction) { + Optional semanticVersion = createSemanticVersion(changeTransaction.getSemVer()); + return semanticVersion.map(version -> Pair.of(version, changeTransaction)); + } + + private static Optional createSemanticVersion(String semVer) { + String truncatedSemVer = truncateSemVer(semVer); + try { + SemanticVersion semanticVersion = SemanticVersion.parse(truncatedSemVer); + return Optional.of(semanticVersion); + } catch (SemanticVersion.SemanticVersionParseException e) { + return Optional.empty(); + } + } + + // The semVer is currently returned from the ChangeTransactions in the format "x.y.z-computed". This function + // removes the suffix "computed". + private static String truncateSemVer(String semVer) { + String suffix = "-computed"; + return semVer.endsWith(suffix) ? semVer.substring(0, semVer.lastIndexOf(suffix)) : semVer; + } + + private static SchemaChange getLastSchemaChange(ChangeEvent changeEvent, long timestamp, String actor, String semVer, + String versionStamp) { + SchemaChange schemaChange = new SchemaChange(); + schemaChange.setTimestampMillis(timestamp); + schemaChange.setActor(actor); + schemaChange.setSemanticVersion(truncateSemVer(semVer)); + schemaChange.setSemanticVersionChange( + SemanticVersionChangeType.valueOf(SemanticVersionChangeType.class, changeEvent.getSemVerChange().toString())); + schemaChange.setChangeType( + ChangeOperationType.valueOf(ChangeOperationType.class, changeEvent.getOperation().toString())); + schemaChange.setVersionStamp(versionStamp); + + String translatedChangeOperationType; + switch (changeEvent.getOperation()) { + case ADD: + translatedChangeOperationType = "Added"; + break; + case MODIFY: + translatedChangeOperationType = "Modified"; + break; + case REMOVE: + translatedChangeOperationType = "Removed"; + break; + default: + translatedChangeOperationType = "Unknown change made"; + break; + } + + String suffix = "-computed"; + String translatedSemVer = semVer.endsWith(suffix) ? semVer.substring(0, semVer.lastIndexOf(suffix)) : semVer; + + String lastSchemaChange = String.format("%s in v%s", translatedChangeOperationType, translatedSemVer); + schemaChange.setLastSchemaChange(lastSchemaChange); + + return schemaChange; + } + + private SchemaBlameMapper() { + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index a1ce09a6d71a1c..1ef4cafe543c66 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -112,6 +112,11 @@ type AppConfig { Configurations related to visual appearance, allows styling the UI without rebuilding the bundle """ visualConfig: VisualConfiguration! + + """ + Configurations related to the Timeline API, including the schema blame view. + """ + timelineConfig: TimelineConfig! } """ diff --git a/datahub-graphql-core/src/main/resources/timeline.graphql b/datahub-graphql-core/src/main/resources/timeline.graphql new file mode 100644 index 00000000000000..ab6404eed6f0db --- /dev/null +++ b/datahub-graphql-core/src/main/resources/timeline.graphql @@ -0,0 +1,146 @@ +extend type Query { + """ + Get list of schema blame + """ + listSchemaBlame(input: ListSchemaBlameInput!): ListSchemaBlameResult +} + +type TimelineConfig { + """ + Whether the timeline feature is enabled. + """ + enabled: Boolean! +} + +""" +Enum of SemanticVersionChange +""" +enum SemanticVersionChangeType { + NONE + PATCH + MAJOR + MINOR + EXCEPTIONAL +} + + +""" +Enum of types of changes +""" +enum ChangeOperationType { + ADD + MODIFY + REMOVE +} + +""" +Enum of CategoryTypes +""" +enum ChangeCategoryType { + DOCUMENTATION + GLOSSARY_TERM + OWNERSHIP + TECHNICAL_SCHEMA + TAG +} + +""" +Input for getting list of latest schema changes computed at a specific version. +""" +input ListSchemaBlameInput { + """ + The dataset urn + """ + datasetUrn: String! + """ + Categories to filter on + """ + categories: [ChangeCategoryType!] + """ + Changes after version cutoff are not shown + """ + versionCutoff: String + +} + +""" +List of latest schema changes computed at a specific version. +""" +type ListSchemaBlameResult { + """ + Latest and current semantic version + """ + latestSemVer: SemVerStruct! + """ + Selected semantic version + """ + selectedSemVer: SemVerStruct + """ + All semantic versions + """ + semVerList: [SemVerStruct!] + """ + List of schema histories + """ + schemaBlameList: [SchemaBlame!] +} + +""" +Schema history +""" +type SchemaBlame { + """ + Flattened name of a field identifying the field the editable info is applied to + """ + fieldPath: String! + """ + Schema change + """ + schemaChange : SchemaChange +} + +type SchemaChange { + """ + The time at which the schema was updated + """ + timestampMillis: Long! + """ + The actor who updated the schema + """ + actor: String! + """ + Semantic version of the change + """ + semanticVersion: String! + """ + Semantic version type + """ + semanticVersionChange: SemanticVersionChangeType! + """ + Version stamp of the change + """ + versionStamp: String! + """ + The type of the change + """ + changeType: ChangeOperationType! + """ + Last column update + """ + lastSchemaChange: String +} + +type SemVerStruct { + """ + Semantic version of the change + """ + semVer: String + """ + Semantic version timestamp + """ + semVerTimestamp: Long + """ + Version stamp of the change + """ + versionStamp: String +} \ No newline at end of file diff --git a/datahub-web-react/codegen.yml b/datahub-web-react/codegen.yml index 42052d6f74ec4c..dfb90bd216cb37 100644 --- a/datahub-web-react/codegen.yml +++ b/datahub-web-react/codegen.yml @@ -7,6 +7,7 @@ schema: - '../datahub-graphql-core/src/main/resources/recommendation.graphql' - '../datahub-graphql-core/src/main/resources/auth.graphql' - '../datahub-graphql-core/src/main/resources/ingestion.graphql' + - '../datahub-graphql-core/src/main/resources/timeline.graphql' config: scalars: Long: number diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx index 1fa416387b42e9..c2f64527aaeb61 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx @@ -1,14 +1,21 @@ import React from 'react'; -import { Button, Typography } from 'antd'; -import { FileTextOutlined, TableOutlined } from '@ant-design/icons'; +import { useHistory, useLocation } from 'react-router-dom'; +import { Button, Popover, Radio, Select, Typography } from 'antd'; +import { CaretDownOutlined, FileTextOutlined, InfoCircleOutlined, TableOutlined } from '@ant-design/icons'; import styled from 'styled-components'; import CustomPagination from './CustomPagination'; import TabToolbar from '../../../../shared/components/styled/TabToolbar'; +import { SemVerStruct } from '../../../../../../types.generated'; +import { toRelativeTimeString } from '../../../../../shared/time/timeUtils'; +import { SchemaBlameViewType } from '../utils/types'; +import { ANTD_GRAY } from '../../../../shared/constants'; +import { navigateToVersionedDatasetUrl } from '../../../../shared/tabs/Dataset/Schema/utils/navigateToVersionedDatasetUrl'; const SchemaHeaderContainer = styled.div` display: flex; - justify-content: flex-end; + justify-content: space-between; padding-bottom: 16px; + width: 100%; `; // TODO(Gabe): undo display: none when dbt/bigquery flickering has been resolved @@ -18,6 +25,23 @@ const ShowVersionButton = styled(Button)` display: none; `; +// Below styles are for buttons on the left side of the Schema Header +const LeftButtonsGroup = styled.div` + &&& { + display: flex; + justify-content: left; + width: 100%; + } +`; + +const RawButton = styled(Button)` + &&& { + display: flex; + margin-right: 10px; + justify-content: left; + } +`; + const KeyButton = styled(Button)<{ $highlighted: boolean }>` border-radius: 8px 0px 0px 8px; font-weight: ${(props) => (props.$highlighted ? '600' : '400')}; @@ -33,6 +57,69 @@ const KeyValueButtonGroup = styled.div` display: inline-block; `; +// Below styles are for buttons on the right side of the Schema Header +const RightButtonsGroup = styled.div` + &&& { + display: flex; + justify-content: right; + margin-top: -6px; + width: 100%; + row-gap: 12px; + } +`; + +const SchemaBlameSelector = styled(Select)` + &&& { + font-weight: 400; + margin-top: 6px; + min-width: 17%; + margin-right: 10px; + border-radius: 0px 8px 8px 0px; + } +`; + +const SchemaBlameSelectorOption = styled(Select.Option)` + &&& { + overflow: visible; + margin-top: 6px; + } +`; + +const BlameRadio = styled(Radio.Group)` + &&& { + margin-top: 6px; + margin-right: 10px; + } +`; + +const BlameRadioButton = styled(Radio.Button)` + &&& { + min-width: 30px; + } +`; + +const CurrentVersionTimestampText = styled(Typography.Text)` + &&& { + line-height: 22px; + margin-top: 10px; + margin-right: 10px; + color: ${ANTD_GRAY[7]}; + } +`; + +const StyledInfoCircleOutlined = styled(InfoCircleOutlined)` + &&& { + margin-top: 12px; + font-size: 20px; + } +`; + +const StyledCaretDownOutlined = styled(CaretDownOutlined)` + &&& { + margin-top: 8px; + } +`; + type Props = { maxVersion?: number; fetchVersions?: (version1: number, version2: number) => void; @@ -44,6 +131,11 @@ type Props = { hasKeySchema: boolean; showKeySchema: boolean; setShowKeySchema: (show: boolean) => void; + lastUpdatedTimeString: string; + schemaBlameSelectValue: string; + semVerList: Array; + schemaBlameView: any; + setSchemaBlameView: any; }; export default function SchemaHeader({ @@ -57,7 +149,14 @@ export default function SchemaHeader({ hasKeySchema, showKeySchema, setShowKeySchema, + lastUpdatedTimeString, + schemaBlameSelectValue, + semVerList, + schemaBlameView, + setSchemaBlameView, }: Props) { + const history = useHistory(); + const location = useLocation(); const onVersionChange = (version1, version2) => { if (version1 === null || version2 === null) { return; @@ -65,13 +164,39 @@ export default function SchemaHeader({ fetchVersions?.(version1 - maxVersion, version2 - maxVersion); }; + const semVerDisplayString = (semVer: SemVerStruct) => { + const semVerTimestampString = + (semVer?.semVerTimestamp && toRelativeTimeString(semVer?.semVerTimestamp)) || 'unknown'; + return `${semVer.semVer} - ${semVerTimestampString}`; + }; + + const renderOptions = () => { + return semVerList.map( + (semVerStruct) => + semVerStruct?.semVer && + semVerStruct?.semVerTimestamp && ( + + {semVerDisplayString(semVerStruct)} + + ), + ); + }; + + const onSchemaBlameToggle = (e) => { + setSchemaBlameView(e.target.value); + }; + + const docLink = 'https://datahubproject.io/docs/dev-guides/timeline/'; return ( {maxVersion > 0 && !editMode && } -
+ {hasRaw && ( - + )} {hasKeySchema && ( @@ -97,11 +222,53 @@ export default function SchemaHeader({ )} {maxVersion > 0 && (editMode ? ( - setEditMode?.(false)}>Version History + setEditMode?.(false)}>Version Blame ) : ( setEditMode?.(true)}>Back ))} -
+ + + {lastUpdatedTimeString} + + + Normal + + + Blame + + + { + const datasetVersion: string = e as string; + navigateToVersionedDatasetUrl({ + location, + history, + datasetVersion, + }); + }} + data-testid="schema-version-selector-dropdown" + suffixIcon={} + > + {renderOptions()} + + + Semantic versions for this view were computed using TECHNICAL_SCHEMA. You can find more + info about how we compute versions + + {' '} + here.{' '} + + + } + > + + + ); diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/utils/types.ts b/datahub-web-react/src/app/entity/dataset/profile/schema/utils/types.ts index 68f0f9c8c96c61..4b9786f899255e 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/utils/types.ts +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/utils/types.ts @@ -8,3 +8,8 @@ export interface ExtendedSchemaFields extends SchemaField { isNewRow?: boolean; isDeletedRow?: boolean; } + +export enum SchemaBlameViewType { + NORMAL, + BLAME, +} diff --git a/datahub-web-react/src/app/entity/shared/constants.ts b/datahub-web-react/src/app/entity/shared/constants.ts index 74a8187bc2f56e..51d785c12fd4ae 100644 --- a/datahub-web-react/src/app/entity/shared/constants.ts +++ b/datahub-web-react/src/app/entity/shared/constants.ts @@ -7,6 +7,7 @@ export const REDESIGN_COLORS = { export const ANTD_GRAY = { 1: '#FFFFFF', 2: '#FAFAFA', + 2.5: '#F8F8F8', 3: '#F5F5F5', 4: '#F0F0F0', 4.5: '#E9E9E9', diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx index c0246f3b447cf6..a7bb991246e605 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx @@ -2,6 +2,10 @@ import { Empty } from 'antd'; import React, { useEffect, useMemo, useState } from 'react'; import styled from 'styled-components'; import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated'; +import { + useListSchemaBlameQuery, + useListSchemaBlameVersionsQuery, +} from '../../../../../../graphql/schemaBlame.generated'; import SchemaEditableContext from '../../../../../shared/SchemaEditableContext'; // import { EditableSchemaFieldInfo, GlobalTagsUpdate } from '../../../../../types.generated'; import SchemaHeader from '../../../../dataset/profile/schema/components/SchemaHeader'; @@ -10,8 +14,13 @@ import { KEY_SCHEMA_PREFIX } from '../../../../dataset/profile/schema/utils/cons import { groupByFieldPath } from '../../../../dataset/profile/schema/utils/utils'; import { ANTD_GRAY } from '../../../constants'; import { useBaseEntity, useEntityData } from '../../../EntityContext'; +import { ChangeCategoryType, SemVerStruct } from '../../../../../../types.generated'; +import { toLocalDateTimeString } from '../../../../../shared/time/timeUtils'; +import { SchemaBlameViewType } from '../../../../dataset/profile/schema/utils/types'; import SchemaTable from './SchemaTable'; +import useGetSemanticVersionFromUrlParams from './utils/useGetSemanticVersionFromUrlParams'; +import { useGetVersionedDatasetQuery } from '../../../../../../graphql/versionedDataset.generated'; const NoSchema = styled(Empty)` color: ${ANTD_GRAY[6]}; @@ -21,11 +30,10 @@ const NoSchema = styled(Empty)` export const SchemaTab = ({ properties }: { properties?: any }) => { const { entityData } = useEntityData(); const baseEntity = useBaseEntity(); - let editMode = true; - if (properties && properties.hasOwnProperty('editMode')) { - editMode = properties.editMode; - } - const { schemaMetadata, editableSchemaMetadata } = entityData || {}; + const maybeEntityData = entityData || {}; + let schemaMetadata: any = maybeEntityData?.schemaMetadata || undefined; + let editableSchemaMetadata: any = maybeEntityData?.editableSchemaMetadata || undefined; + const datasetUrn: string = baseEntity?.dataset?.urn || ''; const usageStats = baseEntity?.dataset?.usageStats; const [showRaw, setShowRaw] = useState(false); const hasRawSchema = useMemo( @@ -49,6 +57,57 @@ export const SchemaTab = ({ properties }: { properties?: any }) => { ); const [showKeySchema, setShowKeySchema] = useState(false); + const [schemaBlameView, setSchemaBlameView] = useState(SchemaBlameViewType.NORMAL); + + const { data: listSchemaBlameVersionsData } = useListSchemaBlameVersionsQuery({ + skip: !datasetUrn, + variables: { + input: { + datasetUrn, + categories: [ChangeCategoryType.TechnicalSchema], + }, + }, + }); + const latestSemVer: string = listSchemaBlameVersionsData?.listSchemaBlame?.latestSemVer?.semVer || ''; + + const showSchemaBlame: boolean = schemaBlameView === SchemaBlameViewType.BLAME; + const semVerList: Array = listSchemaBlameVersionsData?.listSchemaBlame?.semVerList || []; + const versionedDataset = useGetSemanticVersionFromUrlParams(); + const schemaBlameSelectValue = versionedDataset || latestSemVer; + + const schemaBlameSelectedSemVerStruct = semVerList.find((semVer) => semVer.semVer === schemaBlameSelectValue); + const schemaBlameSelectedVersionStamp: string = schemaBlameSelectedSemVerStruct?.versionStamp || ''; + + let editMode = true; + if (schemaBlameSelectValue !== latestSemVer) { + editMode = false; + } else if (properties && properties.hasOwnProperty('editMode')) { + editMode = properties.editMode; + } + + const { data: listSchemaBlameData } = useListSchemaBlameQuery({ + skip: !datasetUrn, + variables: { + input: { + datasetUrn, + versionCutoff: schemaBlameSelectValue, + categories: [ChangeCategoryType.TechnicalSchema], + }, + }, + }); + + const versionedDatasetData = useGetVersionedDatasetQuery({ + skip: !datasetUrn || !schemaBlameSelectedVersionStamp, + variables: { + urn: datasetUrn, + versionStamp: schemaBlameSelectedVersionStamp, + }, + }); + + if (schemaBlameSelectValue !== latestSemVer) { + schemaMetadata = versionedDatasetData?.data?.versionedDataset?.schema || undefined; + editableSchemaMetadata = versionedDatasetData?.data?.versionedDataset?.editableSchemaMetadata || undefined; + } // if there is no value schema, default the selected schema to Key useEffect(() => { @@ -60,6 +119,12 @@ export const SchemaTab = ({ properties }: { properties?: any }) => { return groupByFieldPath(schemaMetadata?.fields, { showKeySchema }); }, [schemaMetadata, showKeySchema]); + const lastUpdatedTimeString = `Reported at ${ + (listSchemaBlameData?.listSchemaBlame?.selectedSemVer?.semVerTimestamp && + toLocalDateTimeString(listSchemaBlameData?.listSchemaBlame?.selectedSemVer?.semVerTimestamp)) || + 'unknown' + }`; + return (
{ hasKeySchema={hasKeySchema} showKeySchema={showKeySchema} setShowKeySchema={setShowKeySchema} + lastUpdatedTimeString={lastUpdatedTimeString} + schemaBlameSelectValue={schemaBlameSelectValue} + semVerList={semVerList} + schemaBlameView={schemaBlameView} + setSchemaBlameView={setSchemaBlameView} /> {/* eslint-disable-next-line no-nested-ternary */} {showRaw ? ( @@ -87,6 +157,8 @@ export const SchemaTab = ({ properties }: { properties?: any }) => { editMode={editMode} editableSchemaMetadata={editableSchemaMetadata} usageStats={usageStats} + listSchemaBlameResult={listSchemaBlameData?.listSchemaBlame} + showSchemaBlame={showSchemaBlame} /> diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx index 46071cc029aad8..dd173224c48962 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx @@ -2,10 +2,10 @@ import React, { useMemo, useState } from 'react'; import { ColumnsType } from 'antd/es/table'; import styled from 'styled-components'; import {} from 'antd'; - import { EditableSchemaMetadata, ForeignKeyConstraint, + ListSchemaBlameResult, SchemaField, SchemaMetadata, UsageQueryResult, @@ -19,6 +19,8 @@ import ExpandIcon from './components/ExpandIcon'; import { StyledTable } from '../../../components/styled/StyledTable'; import { SchemaRow } from './components/SchemaRow'; import { FkContext } from './utils/selectedFkContext'; +import useSchemaBlameRenderer from './utils/useSchemaBlameRenderer'; +import { ANTD_GRAY } from '../../../constants'; const TableContainer = styled.div` &&& .ant-table-tbody > tr > .ant-table-cell-with-append { @@ -41,6 +43,8 @@ export type Props = { editableSchemaMetadata?: EditableSchemaMetadata | null; editMode?: boolean; usageStats?: UsageQueryResult | null; + listSchemaBlameResult?: ListSchemaBlameResult | null; + showSchemaBlame: boolean; }; export default function SchemaTable({ rows, @@ -48,8 +52,11 @@ export default function SchemaTable({ editableSchemaMetadata, usageStats, editMode = true, + listSchemaBlameResult, + showSchemaBlame, }: Props): JSX.Element { const hasUsageStats = useMemo(() => (usageStats?.aggregations?.fields?.length || 0) > 0, [usageStats]); + const schemaBlameList = listSchemaBlameResult?.schemaBlameList || []; const [tagHoveredIndex, setTagHoveredIndex] = useState(undefined); const [selectedFkFieldPath, setSelectedFkFieldPath] = @@ -66,6 +73,7 @@ export default function SchemaTable({ showTerms: true, }); const schemaTitleRenderer = useSchemaTitleRenderer(schemaMetadata, setSelectedFkFieldPath); + const schemaBlameRenderer = useSchemaBlameRenderer(schemaBlameList); const onTagTermCell = (record: SchemaField, rowIndex: number | undefined) => ({ onMouseEnter: () => { @@ -122,12 +130,30 @@ export default function SchemaTable({ width: 300, }; + const blameColumn = { + dataIndex: 'fieldPath', + key: 'fieldPath', + width: 75, + render(record: SchemaField) { + return { + props: { + style: { backgroundColor: ANTD_GRAY[2.5] }, + }, + children: schemaBlameRenderer(record), + }; + }, + }; + let allColumns: ColumnsType = [fieldColumn, descriptionColumn, tagColumn, termColumn]; if (hasUsageStats) { allColumns = [...allColumns, usageColumn]; } + if (showSchemaBlame) { + allColumns = [...allColumns, blameColumn]; + } + return ( diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaRow.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaRow.tsx index d6023cc064b2be..075bd8f1c598b6 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaRow.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaRow.tsx @@ -102,7 +102,7 @@ export const SchemaRow = ({ return ( <> {children} - {fieldPath === selectedFk?.fieldPath && ( + {fieldPath && fieldPath === selectedFk?.fieldPath && ( @@ -110,7 +110,7 @@ export const SchemaRow = ({ {selectedFk.constraint?.foreignDataset?.name} @@ -121,14 +121,14 @@ export const SchemaRow = ({ {entityRegistry.renderProfile( EntityType.Dataset, - selectedFk.constraint?.foreignDataset?.urn || '', + selectedFk?.constraint?.foreignDataset?.urn || '', )}
{baseEntity.dataset?.name} - {selectedFk.constraint?.sourceFields?.map((field) => ( + {selectedFk?.constraint?.sourceFields?.map((field) => (
@@ -136,8 +136,8 @@ export const SchemaRow = ({
{'--->'}
- {selectedFk.constraint?.foreignDataset?.name} - {selectedFk.constraint?.foreignFields?.map((field) => ( + {selectedFk?.constraint?.foreignDataset?.name} + {selectedFk?.constraint?.foreignFields?.map((field) => (
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/navigateToVersionedDatasetUrl.ts b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/navigateToVersionedDatasetUrl.ts new file mode 100644 index 00000000000000..0ad89e5fd6230e --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/navigateToVersionedDatasetUrl.ts @@ -0,0 +1,27 @@ +import * as QueryString from 'query-string'; +import { RouteComponentProps } from 'react-router-dom'; + +export const navigateToVersionedDatasetUrl = ({ + location, + history, + datasetVersion, +}: { + location: { + search: string; + pathname: string; + }; + history: RouteComponentProps['history']; + datasetVersion: string; +}) => { + const parsedSearch = QueryString.parse(location.search, { arrayFormat: 'comma' }); + const newSearch = { + ...parsedSearch, + semantic_version: datasetVersion, + }; + const newSearchStringified = QueryString.stringify(newSearch, { arrayFormat: 'comma' }); + + history.push({ + pathname: location.pathname, + search: newSearchStringified, + }); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useGetSemanticVersionFromUrlParams.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useGetSemanticVersionFromUrlParams.tsx new file mode 100644 index 00000000000000..3b8a13141b52de --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useGetSemanticVersionFromUrlParams.tsx @@ -0,0 +1,9 @@ +import * as QueryString from 'query-string'; +import { useLocation } from 'react-router-dom'; + +export default function useGetSemanticVersionFromUrlParams() { + const location = useLocation(); + const params = QueryString.parse(location.search, { arrayFormat: 'comma' }); + const semanticVersion: string = params.semantic_version as string; + return semanticVersion; +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useSchemaBlameRenderer.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useSchemaBlameRenderer.tsx new file mode 100644 index 00000000000000..bfb4187145eb46 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useSchemaBlameRenderer.tsx @@ -0,0 +1,95 @@ +import React from 'react'; +import { useHistory, useLocation } from 'react-router-dom'; +import { Button, Tooltip, Typography } from 'antd'; +import { ScanOutlined } from '@ant-design/icons'; +import styled from 'styled-components'; +import { SchemaField, SchemaBlame } from '../../../../../../../types.generated'; +import { pathMatchesNewPath } from '../../../../../dataset/profile/schema/utils/utils'; +import { toRelativeTimeString } from '../../../../../../shared/time/timeUtils'; +import { navigateToVersionedDatasetUrl } from './navigateToVersionedDatasetUrl'; + +const HeadingDiv = styled.div` + vertical-align: top; + display: flex; + flex-direction: column; + margin-top: 16px; +`; + +const SubheadingDiv = styled.div` + display: flex; + flex-wrap: wrap; + justify-content: space-between; +`; + +const SchemaBlameText = styled(Typography.Text)` + font-size: 14x; + line-height: 22px; + font-family: 'Roboto Mono', monospace; + font-weight: 500; +`; + +const SchemaBlameTimestampText = styled(Typography.Text)` + font-size: 8px; + line-height: 22px; + font-family: 'Roboto Mono', monospace; + font-weight: 500; +`; + +const SchemaBlameBlameButton = styled(Button)` + display: inline-block; + width: 30px; +`; + +export default function useSchemaBlameRenderer(schemaBlameList?: Array | null) { + const history = useHistory(); + const location = useLocation(); + const schemaBlameRenderer = (record: SchemaField) => { + const relevantSchemaBlame = schemaBlameList?.find((candidateSchemaBlame) => + pathMatchesNewPath(candidateSchemaBlame.fieldPath, String(record)), + ); + + if (!relevantSchemaBlame || !relevantSchemaBlame.schemaChange) { + return null; + } + + const relevantSchemaBlameSemVer = relevantSchemaBlame.schemaChange.semanticVersion; + + return ( + <> + + + {relevantSchemaBlame?.schemaChange?.lastSchemaChange} + + + {relevantSchemaBlame?.schemaChange?.timestampMillis ? ( + + {toRelativeTimeString(relevantSchemaBlame?.schemaChange?.timestampMillis)} + + ) : ( + 'unknown' + )} + + + { + navigateToVersionedDatasetUrl({ + location, + history, + datasetVersion: relevantSchemaBlameSemVer, + }); + }} + size="small" + type="text" + > + + + + + + + + ); + }; + return schemaBlameRenderer; +} diff --git a/datahub-web-react/src/app/shared/time/timeUtils.tsx b/datahub-web-react/src/app/shared/time/timeUtils.tsx index a9f38fa8434f2f..e5d9c50f5983a9 100644 --- a/datahub-web-react/src/app/shared/time/timeUtils.tsx +++ b/datahub-web-react/src/app/shared/time/timeUtils.tsx @@ -10,6 +10,16 @@ export const INTERVAL_TO_SECONDS = { [DateInterval.Year]: 31536000, }; +export const INTERVAL_TO_MS = { + [DateInterval.Second]: 1000, + [DateInterval.Minute]: 60000, + [DateInterval.Hour]: 3600000, + [DateInterval.Day]: 86400000, + [DateInterval.Week]: 604800000, + [DateInterval.Month]: 2419200000, + [DateInterval.Year]: 31536000000, +}; + export type TimeWindowSize = { interval: DateInterval; count: number; @@ -96,3 +106,42 @@ export const toUTCDateTimeString = (timeMs: number) => { export const getLocaleTimezone = () => { return Intl.DateTimeFormat().resolvedOptions().timeZone; }; + +export const toRelativeTimeString = (timeMs: number) => { + const rtf = new Intl.RelativeTimeFormat('en', { numeric: 'auto' }); + + const diffInMs = timeMs - new Date().getTime(); + + const diffInSeconds = Math.round(diffInMs / INTERVAL_TO_MS[DateInterval.Second]); + if (Math.abs(diffInSeconds) > 0 && Math.abs(diffInSeconds) <= 60) { + return rtf.format(diffInSeconds, 'second'); + } + + const diffInMinutes = Math.round(diffInMs / INTERVAL_TO_MS[DateInterval.Minute]); + if (Math.abs(diffInMinutes) > 0 && Math.abs(diffInMinutes) <= 60) { + return rtf.format(diffInMinutes, 'minute'); + } + + const diffInHours = Math.round(diffInMs / INTERVAL_TO_MS[DateInterval.Hour]); + if (Math.abs(diffInHours) > 0 && Math.abs(diffInHours) <= 24) { + return rtf.format(diffInHours, 'hour'); + } + + const diffInDays = Math.round(diffInMs / INTERVAL_TO_MS[DateInterval.Day]); + if (Math.abs(diffInDays) > 0 && Math.abs(diffInDays) <= 7) { + return rtf.format(diffInDays, 'day'); + } + + const diffInWeeks = Math.round(diffInMs / INTERVAL_TO_MS[DateInterval.Week]); + if (Math.abs(diffInWeeks) > 0 && Math.abs(diffInWeeks) <= 4) { + return rtf.format(diffInWeeks, 'week'); + } + + const diffInMonths = Math.round(diffInMs / INTERVAL_TO_MS[DateInterval.Month]); + if (Math.abs(diffInMonths) > 0 && Math.abs(diffInMonths) <= 12) { + return rtf.format(diffInMonths, 'month'); + } + + const diffInYears = Math.round(diffInMs / INTERVAL_TO_MS[DateInterval.Year]); + return rtf.format(diffInYears, 'year'); +}; diff --git a/datahub-web-react/src/appConfigContext.tsx b/datahub-web-react/src/appConfigContext.tsx index 0c0b3bde628b66..c402a575872031 100644 --- a/datahub-web-react/src/appConfigContext.tsx +++ b/datahub-web-react/src/appConfigContext.tsx @@ -25,6 +25,9 @@ export const DEFAULT_APP_CONFIG = { authConfig: { tokenAuthEnabled: false, }, + timelineConfig: { + enabled: false, + }, }; export const AppConfigContext = React.createContext<{ diff --git a/datahub-web-react/src/graphql/app.graphql b/datahub-web-react/src/graphql/app.graphql index 40f7cba9d69d50..06193e54d9abea 100644 --- a/datahub-web-react/src/graphql/app.graphql +++ b/datahub-web-react/src/graphql/app.graphql @@ -37,6 +37,9 @@ query appConfig { visualConfig { logoUrl } + timelineConfig { + enabled + } } } diff --git a/datahub-web-react/src/graphql/schemaBlame.graphql b/datahub-web-react/src/graphql/schemaBlame.graphql new file mode 100644 index 00000000000000..28e7fd9023e208 --- /dev/null +++ b/datahub-web-react/src/graphql/schemaBlame.graphql @@ -0,0 +1,46 @@ +query listSchemaBlame($input: ListSchemaBlameInput!) { + listSchemaBlame(input: $input) { + latestSemVer { + semVer + semVerTimestamp + versionStamp + } + selectedSemVer { + semVer + semVerTimestamp + versionStamp + } + semVerList { + semVer + semVerTimestamp + versionStamp + } + schemaBlameList { + fieldPath + schemaChange { + timestampMillis + actor + semanticVersion + semanticVersionChange + changeType + lastSchemaChange + versionStamp + } + } + } +} + +query listSchemaBlameVersions($input: ListSchemaBlameInput!) { + listSchemaBlame(input: $input) { + latestSemVer { + semVer + semVerTimestamp + versionStamp + } + semVerList { + semVer + semVerTimestamp + versionStamp + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/examples/mce_files/bootstrap_mce.json b/metadata-ingestion/examples/mce_files/bootstrap_mce.json index 7558c88b8f8d8f..b7b41df66398eb 100644 --- a/metadata-ingestion/examples/mce_files/bootstrap_mce.json +++ b/metadata-ingestion/examples/mce_files/bootstrap_mce.json @@ -375,36 +375,6 @@ "nativeDataType": "varchar(100)", "recursive": false }, - { - "fieldPath": "shipment_info.date", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info date description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "Date", - "recursive": false - }, - { - "fieldPath": "shipment_info.target", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info target description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false - }, { "fieldPath": "shipment_info.destination", "jsonPath": null, @@ -417,7 +387,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "varchar(100)", + "nativeDataType": "text", "recursive": false }, { diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java index 6538f4af21fc78..8302810a043556 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java @@ -20,6 +20,7 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.recommendation.RecommendationsService; import com.linkedin.metadata.secret.SecretService; +import com.linkedin.metadata.timeline.TimelineService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.version.GitVersion; @@ -97,6 +98,10 @@ public class GraphQLEngineFactory { @Qualifier("visualConfig") private VisualConfiguration _visualConfiguration; + @Autowired + @Qualifier("timelineService") + private TimelineService _timelineService; + @Value("${platformAnalytics.enabled}") // TODO: Migrate to DATAHUB_ANALYTICS_ENABLED private Boolean isAnalyticsEnabled; @@ -119,6 +124,7 @@ protected GraphQLEngine getInstance() { _configProvider.getAuthentication(), _configProvider.getAuthorization(), _gitVersion, + _timelineService, _graphService.supportsMultiHop(), _visualConfiguration ).builder().build(); @@ -138,6 +144,7 @@ protected GraphQLEngine getInstance() { _configProvider.getAuthentication(), _configProvider.getAuthorization(), _gitVersion, + _timelineService, _graphService.supportsMultiHop(), _visualConfiguration ).builder().build(); diff --git a/smoke-test/tests/cypress/cypress/integration/schema_blame/schema_blame.js b/smoke-test/tests/cypress/cypress/integration/schema_blame/schema_blame.js new file mode 100644 index 00000000000000..6e282b52496369 --- /dev/null +++ b/smoke-test/tests/cypress/cypress/integration/schema_blame/schema_blame.js @@ -0,0 +1,53 @@ +describe('schema blame', () => { + Cypress.on('uncaught:exception', (err, runnable) => { + return false; + }); + + it('can activate the blame view and verify for the latest version of a dataset', () => { + cy.login(); + cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)/Schema'); + cy.wait(10000); + + // Verify which fields are present, along with checking descriptions and tags + cy.contains('field_foo'); + cy.contains('field_baz'); + cy.contains('field_bar').should('not.exist'); + cy.contains('Foo field description has changed'); + cy.contains('Baz field description'); + cy.get('[data-testid="schema-field-field_foo-tags"]').contains('Legacy'); + + // Make sure the schema blame is accurate + cy.get('[data-testid="schema-blame-button"]').click({ force: true }); + cy.wait(3000); + + cy.get('[data-testid="field_foo-schema-blame-description"]').contains("Modified in v1.0.0"); + cy.get('[data-testid="field_baz-schema-blame-description"]').contains("Added in v1.0.0"); + + // Verify the "view blame prior to" button changes state by modifying the URL + cy.get('[data-testid="field_foo-view-prior-blame-button"]').click({force: true}); + cy.wait(3000); + + cy.url().should('include', 'semantic_version=1.0.0'); + }); + + it('can activate the blame view and verify for an older version of a dataset', () => { + cy.login(); + cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)/Schema?semantic_version=0.0.0'); + cy.wait(10000); + + // Verify which fields are present, along with checking descriptions and tags + cy.contains('field_foo'); + cy.contains('field_bar'); + cy.contains('field_baz').should('not.exist'); + cy.contains('Foo field description'); + cy.contains('Bar field description'); + cy.get('[data-testid="schema-field-field_foo-tags"]').contains('Legacy').should('not.exist'); + + // Make sure the schema blame is accurate + cy.get('[data-testid="schema-blame-button"]').click({ force: true }); + cy.wait(3000); + + cy.get('[data-testid="field_foo-schema-blame-description"]').contains("Added in v0.0.0"); + cy.get('[data-testid="field_bar-schema-blame-description"]').contains("Added in v0.0.0"); +}); +}) \ No newline at end of file diff --git a/smoke-test/tests/cypress/integration_test.py b/smoke-test/tests/cypress/integration_test.py index de2a4eac901220..848deb9c005ede 100644 --- a/smoke-test/tests/cypress/integration_test.py +++ b/smoke-test/tests/cypress/integration_test.py @@ -9,9 +9,11 @@ def ingest_cleanup_data(): print("ingesting test data") ingest_file_via_rest("tests/cypress/data.json") + ingest_file_via_rest("tests/cypress/schema-blame-data.json") yield print("removing test data") delete_urns_from_file("tests/cypress/data.json") + delete_urns_from_file("tests/cypress/schema-blame-data.json") def test_run_cypress(frontend_session, wait_for_healthchecks): diff --git a/smoke-test/tests/cypress/schema-blame-data.json b/smoke-test/tests/cypress/schema-blame-data.json new file mode 100644 index 00000000000000..e2dd3bbd1c6032 --- /dev/null +++ b/smoke-test/tests/cypress/schema-blame-data.json @@ -0,0 +1,152 @@ +[ + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.EditableSchemaMetadata": { + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "editableSchemaFieldInfo": [ + { + "fieldPath": "field_foo", + "description": "Foo field description has changed", + "globalTags": { "tags": [{ "tag": "urn:li:tag:Legacy" }] } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "fields": [ + { + "fieldPath": "field_foo", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Foo field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "field_baz", + "jsonPath": null, + "nullable": false, + "description": { + "string": "Baz field description" + }, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [{ "tag": "urn:li:tag:Cypress" }] + } + } + ] + } + }, + "proposedDelta": null + } +]