Merge branch 'datahub-project:master' into master

.github/pr-labeler-config.yml

@@ -1,9 +1,13 @@
 ingestion:
   - "metadata-ingestion/**/*"
+  - "metadata-ingestion-modules/**/*"
+  - "metadata-integration/**/*"
 
 devops:
   - "docker/**/*"
   - ".github/**/*"
+  - "perf-test/**/*"
+  - "metadata-service/**/*"
 
 product:
   - "datahub-web-react/**/*"

datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java

@@ -15,6 +15,7 @@
 import com.nimbusds.oauth2.sdk.http.HTTPRequest;
 import com.nimbusds.oauth2.sdk.http.HTTPResponse;
 import com.nimbusds.oauth2.sdk.id.ClientID;
+import com.nimbusds.oauth2.sdk.pkce.CodeVerifier;
 import com.nimbusds.openid.connect.sdk.OIDCTokenResponse;
 import com.nimbusds.openid.connect.sdk.OIDCTokenResponseParser;
 import com.nimbusds.openid.connect.sdk.token.OIDCTokens;
@@ -144,8 +145,10 @@ public void validate(final OidcCredentials credentials, final WebContext context
     if (code != null) {
       try {
         final String computedCallbackUrl = client.computeFinalCallbackUrl(context);
+        CodeVerifier verifier = (CodeVerifier) configuration.getValueRetriever()
+                .retrieve(client.getCodeVerifierSessionAttributeName(), client, context).orElse(null);
         // Token request
-        final TokenRequest request = createTokenRequest(new AuthorizationCodeGrant(code, new URI(computedCallbackUrl)));
+        final TokenRequest request = createTokenRequest(new AuthorizationCodeGrant(code, new URI(computedCallbackUrl), verifier));
         HTTPRequest tokenHttpRequest = request.toHTTPRequest();
         tokenHttpRequest.setConnectTimeout(configuration.getConnectTimeout());
         tokenHttpRequest.setReadTimeout(configuration.getReadTimeout());

datahub-graphql-core/build.gradle

@@ -36,6 +36,7 @@ graphqlCodegen {
         "$projectDir/src/main/resources/timeline.graphql".toString(),
         "$projectDir/src/main/resources/tests.graphql".toString(),
         "$projectDir/src/main/resources/step.graphql".toString(),
+        "$projectDir/src/main/resources/lineage.graphql".toString(),
     ]
     outputDir = new File("$projectDir/src/mainGeneratedGraphQL/java")
     packageName = "com.linkedin.datahub.graphql.generated"

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java

@@ -19,6 +19,7 @@ public class Constants {
     public static final String TIMELINE_SCHEMA_FILE = "timeline.graphql";
     public static final String TESTS_SCHEMA_FILE = "tests.graphql";
     public static final String STEPS_SCHEMA_FILE = "step.graphql";
+    public static final String LINEAGE_SCHEMA_FILE = "lineage.graphql";
     public static final String BROWSE_PATH_DELIMITER = "/";
     public static final String VERSION_STAMP_FIELD_NAME = "versionStamp";
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -3,11 +3,11 @@
 import com.datahub.authentication.AuthenticationConfiguration;
 import com.datahub.authentication.group.GroupService;
 import com.datahub.authentication.invite.InviteTokenService;
+import com.datahub.authentication.post.PostService;
 import com.datahub.authentication.token.StatefulTokenService;
 import com.datahub.authentication.user.NativeUserService;
 import com.datahub.authorization.AuthorizationConfiguration;
 import com.datahub.authorization.role.RoleService;
-import com.datahub.authentication.post.PostService;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.VersionedUrn;
 import com.linkedin.common.urn.Urn;
@@ -143,6 +143,7 @@
 import com.linkedin.datahub.graphql.resolvers.ingest.source.UpsertIngestionSourceResolver;
 import com.linkedin.datahub.graphql.resolvers.jobs.DataJobRunsResolver;
 import com.linkedin.datahub.graphql.resolvers.jobs.EntityRunsResolver;
+import com.linkedin.datahub.graphql.resolvers.lineage.UpdateLineageResolver;
 import com.linkedin.datahub.graphql.resolvers.load.AspectResolver;
 import com.linkedin.datahub.graphql.resolvers.load.BatchGetEntitiesResolver;
 import com.linkedin.datahub.graphql.resolvers.load.EntityLineageResultResolver;
@@ -179,13 +180,13 @@
 import com.linkedin.datahub.graphql.resolvers.mutate.UpdateNameResolver;
 import com.linkedin.datahub.graphql.resolvers.mutate.UpdateParentNodeResolver;
 import com.linkedin.datahub.graphql.resolvers.mutate.UpdateUserSettingResolver;
-import com.linkedin.datahub.graphql.resolvers.settings.user.UpdateCorpUserViewsSettingsResolver;
 import com.linkedin.datahub.graphql.resolvers.operation.ReportOperationResolver;
 import com.linkedin.datahub.graphql.resolvers.policy.DeletePolicyResolver;
 import com.linkedin.datahub.graphql.resolvers.policy.GetGrantedPrivilegesResolver;
 import com.linkedin.datahub.graphql.resolvers.policy.ListPoliciesResolver;
 import com.linkedin.datahub.graphql.resolvers.policy.UpsertPolicyResolver;
 import com.linkedin.datahub.graphql.resolvers.post.CreatePostResolver;
+import com.linkedin.datahub.graphql.resolvers.post.DeletePostResolver;
 import com.linkedin.datahub.graphql.resolvers.post.ListPostsResolver;
 import com.linkedin.datahub.graphql.resolvers.recommendation.ListRecommendationsResolver;
 import com.linkedin.datahub.graphql.resolvers.role.AcceptRoleResolver;
@@ -198,10 +199,11 @@
 import com.linkedin.datahub.graphql.resolvers.search.SearchAcrossEntitiesResolver;
 import com.linkedin.datahub.graphql.resolvers.search.SearchAcrossLineageResolver;
 import com.linkedin.datahub.graphql.resolvers.search.SearchResolver;
-import com.linkedin.datahub.graphql.resolvers.step.BatchGetStepStatesResolver;
-import com.linkedin.datahub.graphql.resolvers.step.BatchUpdateStepStatesResolver;
+import com.linkedin.datahub.graphql.resolvers.settings.user.UpdateCorpUserViewsSettingsResolver;
 import com.linkedin.datahub.graphql.resolvers.settings.view.GlobalViewsSettingsResolver;
 import com.linkedin.datahub.graphql.resolvers.settings.view.UpdateGlobalViewsSettingsResolver;
+import com.linkedin.datahub.graphql.resolvers.step.BatchGetStepStatesResolver;
+import com.linkedin.datahub.graphql.resolvers.step.BatchUpdateStepStatesResolver;
 import com.linkedin.datahub.graphql.resolvers.tag.CreateTagResolver;
 import com.linkedin.datahub.graphql.resolvers.tag.DeleteTagResolver;
 import com.linkedin.datahub.graphql.resolvers.tag.SetTagColorResolver;
@@ -278,6 +280,7 @@
 import com.linkedin.metadata.secret.SecretService;
 import com.linkedin.metadata.service.SettingsService;
 import com.linkedin.metadata.service.ViewService;
+import com.linkedin.metadata.service.LineageService;
 import com.linkedin.metadata.telemetry.TelemetryConfiguration;
 import com.linkedin.metadata.timeline.TimelineService;
 import com.linkedin.metadata.timeseries.TimeseriesAspectService;
@@ -339,6 +342,7 @@ public class GmsGraphQLEngine {
     private final PostService postService;
     private final SettingsService settingsService;
     private final ViewService viewService;
+    private final LineageService lineageService;
 
     private final FeatureFlags featureFlags;
 
@@ -421,7 +425,7 @@ public GmsGraphQLEngine(final EntityClient entityClient, final GraphClient graph
         final GroupService groupService, final RoleService roleService,
         final InviteTokenService inviteTokenService, final PostService postService,
         final ViewService viewService,
-        final SettingsService settingsService,
+        final SettingsService settingsService, final LineageService lineageService,
         final FeatureFlags featureFlags) {
 
         this.entityClient = entityClient;
@@ -446,6 +450,7 @@ public GmsGraphQLEngine(final EntityClient entityClient, final GraphClient graph
         this.postService = postService;
         this.viewService = viewService;
         this.settingsService = settingsService;
+        this.lineageService = lineageService;
 
         this.ingestionConfiguration = Objects.requireNonNull(ingestionConfiguration);
         this.authenticationConfiguration = Objects.requireNonNull(authenticationConfiguration);
@@ -592,6 +597,7 @@ public GraphQLEngine.Builder builder() {
             .addSchema(fileBasedSchema(TIMELINE_SCHEMA_FILE))
             .addSchema(fileBasedSchema(TESTS_SCHEMA_FILE))
             .addSchema(fileBasedSchema(STEPS_SCHEMA_FILE))
+            .addSchema(fileBasedSchema(LINEAGE_SCHEMA_FILE))
             .addDataLoaders(loaderSuppliers(loadableTypes))
             .addDataLoader("Aspect", context -> createDataLoader(aspectType, context))
             .configureRuntimeWiring(this::configureRuntimeWiring);
@@ -854,12 +860,14 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
             .dataFetcher("createInviteToken", new CreateInviteTokenResolver(this.inviteTokenService))
             .dataFetcher("acceptRole", new AcceptRoleResolver(this.roleService, this.inviteTokenService))
             .dataFetcher("createPost", new CreatePostResolver(this.postService))
+            .dataFetcher("deletePost", new DeletePostResolver(this.postService))
             .dataFetcher("batchUpdateStepStates", new BatchUpdateStepStatesResolver(this.entityClient))
             .dataFetcher("createView", new CreateViewResolver(this.viewService))
             .dataFetcher("updateView", new UpdateViewResolver(this.viewService))
             .dataFetcher("deleteView", new DeleteViewResolver(this.viewService))
             .dataFetcher("updateGlobalViewsSettings", new UpdateGlobalViewsSettingsResolver(this.settingsService))
             .dataFetcher("updateCorpUserViewsSettings", new UpdateCorpUserViewsSettingsResolver(this.settingsService))
+            .dataFetcher("updateLineage", new UpdateLineageResolver(this.entityService, this.lineageService))
         );
     }
 
@@ -896,6 +904,13 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder
             .type("LineageRelationship", typeWiring -> typeWiring
                 .dataFetcher("entity", new EntityTypeResolver(entityTypes,
                         (env) -> ((LineageRelationship) env.getSource()).getEntity()))
+                .dataFetcher("createdActor",
+                    new EntityTypeResolver(entityTypes,
+                        (env) -> {
+                            final LineageRelationship relationship = env.getSource();
+                            return relationship.getCreatedActor() != null ? relationship.getCreatedActor() : null;
+                        })
+                )
             )
             .type("ListDomainsResult", typeWiring -> typeWiring
                 .dataFetcher("domains", new LoadableTypeBatchResolver<>(domainType,
@@ -988,6 +1003,7 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) {
                    "dataset",
                    "subTypes"))
                 .dataFetcher("runs", new EntityRunsResolver(entityClient))
+                .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
                 .dataFetcher("parentContainers", new ParentContainersResolver(entityClient)))
             .type("Owner", typeWiring -> typeWiring
                     .dataFetcher("owner", new OwnerTypeResolver<>(ownerTypes,
@@ -1182,6 +1198,7 @@ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) {
             .dataFetcher("parentContainers", new ParentContainersResolver(entityClient))
             .dataFetcher("usageStats", new DashboardUsageStatsResolver(timeseriesAspectService))
             .dataFetcher("statsSummary", new DashboardStatsSummaryResolver(timeseriesAspectService))
+            .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
         );
         builder.type("DashboardInfo", typeWiring -> typeWiring
             .dataFetcher("charts", new LoadableTypeBatchResolver<>(chartType,
@@ -1233,6 +1250,7 @@ private void configureChartResolvers(final RuntimeWiring.Builder builder) {
             )
             .dataFetcher("parentContainers", new ParentContainersResolver(entityClient))
             .dataFetcher("statsSummary", new ChartStatsSummaryResolver(this.timeseriesAspectService))
+            .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
         );
         builder.type("ChartInfo", typeWiring -> typeWiring
             .dataFetcher("inputs", new LoadableTypeBatchResolver<>(datasetType,
@@ -1309,6 +1327,7 @@ private void configureDataJobResolvers(final RuntimeWiring.Builder builder) {
                         })
                 )
                 .dataFetcher("runs", new DataJobRunsResolver(entityClient))
+                .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
             )
             .type("DataJobInputOutput", typeWiring -> typeWiring
                 .dataFetcher("inputDatasets", new LoadableTypeBatchResolver<>(datasetType,

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/entity/EntityPrivilegesResolver.java

@@ -1,17 +1,23 @@
 package com.linkedin.datahub.graphql.resolvers.entity;
 
+import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.datahub.graphql.QueryContext;
+import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
+import com.linkedin.datahub.graphql.authorization.ConjunctivePrivilegeGroup;
+import com.linkedin.datahub.graphql.authorization.DisjunctivePrivilegeGroup;
 import com.linkedin.datahub.graphql.generated.Entity;
 import com.linkedin.datahub.graphql.generated.EntityPrivileges;
 import com.linkedin.datahub.graphql.resolvers.mutate.util.GlossaryUtils;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.Constants;
+import com.linkedin.metadata.authorization.PoliciesConfig;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
 import lombok.extern.slf4j.Slf4j;
 
+import java.util.Collections;
 import java.util.concurrent.CompletableFuture;
 
 @Slf4j
@@ -35,6 +41,14 @@ public CompletableFuture<EntityPrivileges> get(DataFetchingEnvironment environme
           return getGlossaryTermPrivileges(urn, context);
         case Constants.GLOSSARY_NODE_ENTITY_NAME:
           return getGlossaryNodePrivileges(urn, context);
+        case Constants.DATASET_ENTITY_NAME:
+          return getDatasetPrivileges(urn, context);
+        case Constants.CHART_ENTITY_NAME:
+          return getChartPrivileges(urn, context);
+        case Constants.DASHBOARD_ENTITY_NAME:
+          return getDashboardPrivileges(urn, context);
+        case Constants.DATA_JOB_ENTITY_NAME:
+          return getDataJobPrivileges(urn, context);
         default:
           log.warn("Tried to get entity privileges for entity type {} but nothing is implemented for it yet", urn.getEntityType());
           return new EntityPrivileges();
@@ -75,4 +89,45 @@ private EntityPrivileges getGlossaryNodePrivileges(Urn nodeUrn, QueryContext con
     }
     return result;
   }
+
+  private boolean canEditEntityLineage(Urn urn, QueryContext context) {
+    final ConjunctivePrivilegeGroup allPrivilegesGroup = new ConjunctivePrivilegeGroup(ImmutableList.of(
+        PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType()
+    ));
+    DisjunctivePrivilegeGroup orPrivilegesGroup = new DisjunctivePrivilegeGroup(ImmutableList.of(
+        allPrivilegesGroup,
+        new ConjunctivePrivilegeGroup(Collections.singletonList(PoliciesConfig.EDIT_LINEAGE_PRIVILEGE.getType()))
+    ));
+
+    return AuthorizationUtils.isAuthorized(
+        context.getAuthorizer(),
+        context.getActorUrn(),
+        urn.getEntityType(),
+        urn.toString(),
+        orPrivilegesGroup);
+  }
+
+  private EntityPrivileges getDatasetPrivileges(Urn urn, QueryContext context) {
+    final EntityPrivileges result = new EntityPrivileges();
+    result.setCanEditLineage(canEditEntityLineage(urn, context));
+    return result;
+  }
+
+  private EntityPrivileges getChartPrivileges(Urn urn, QueryContext context) {
+    final EntityPrivileges result = new EntityPrivileges();
+    result.setCanEditLineage(canEditEntityLineage(urn, context));
+    return result;
+  }
+
+  private EntityPrivileges getDashboardPrivileges(Urn urn, QueryContext context) {
+    final EntityPrivileges result = new EntityPrivileges();
+    result.setCanEditLineage(canEditEntityLineage(urn, context));
+    return result;
+  }
+
+  private EntityPrivileges getDataJobPrivileges(Urn urn, QueryContext context) {
+    final EntityPrivileges result = new EntityPrivileges();
+    result.setCanEditLineage(canEditEntityLineage(urn, context));
+    return result;
+  }
 }