Merge branch 'master' into ge-profile-cardinality

.github/pr-labeler-config.yml

@@ -13,3 +13,6 @@ product:
 
 docs:
   - "docs/**/*"
+
+smoke_test:
+  - "smoke-test/**/*"

.gitignore

@@ -74,3 +74,5 @@ temp/**
 datahub-frontend/public/**
 
 .remote*
+# Ignore runtime generated authenticatior/authorizer jar files
+metadata-service/plugin/src/test/resources/sample-plugins/**

README.md

@@ -110,8 +110,11 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 - [Adevinta](https://www.adevinta.com/)
 - [Banksalad](https://www.banksalad.com)
 - [Cabify](https://cabify.tech/)
+- [ClassDojo](https://www.classdojo.com/)
+- [Coursera](https://www.coursera.org/)
 - [DefinedCrowd](http://www.definedcrowd.com)
 - [DFDS](https://www.dfds.com/)
+- [Digital Turbine](https://www.digitalturbine.com/)
 - [Expedia Group](http://expedia.com)
 - [Experius](https://www.experius.nl)
 - [Geotab](https://www.geotab.com)
@@ -122,15 +125,21 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 - [Klarna](https://www.klarna.com)
 - [LinkedIn](http://linkedin.com)
 - [Moloco](https://www.moloco.com/en)
+- [N26](https://n26brasil.com/)
+- [Optum](https://www.optum.com/)
 - [Peloton](https://www.onepeloton.com)
 - [Razer](https://www.razer.com)
 - [Saxo Bank](https://www.home.saxo)
+- [Showroomprive](https://www.showroomprive.com/)
+- [SpotHero](https://spothero.com)
 - [Stash](https://www.stash.com)
 - [Shanghai HuaRui Bank](https://www.shrbank.com)
 - [ThoughtWorks](https://www.thoughtworks.com)
 - [TypeForm](http://typeform.com)
+- [Udemy](https://www.udemy.com/)
 - [Uphold](https://uphold.com)
 - [Viasat](https://viasat.com)
+- [Wikimedia](https://www.wikimedia.org)
 - [Wolt](https://wolt.com)
 - [Zynga](https://www.zynga.com)
 

build.gradle

@@ -8,6 +8,12 @@ buildscript {
   ext.neo4jVersion = '4.4.9'
   ext.graphQLJavaVersion = '19.0'
   ext.testContainersVersion = '1.17.4'
+  ext.jacksonVersion = '2.13.4'
+  ext.jettyVersion = '9.4.46.v20220331'
+  ext.log4jVersion = '2.19.0'
+  ext.slf4jVersion = '1.7.32'
+  ext.logbackClassic = '1.2.11'
+
   apply from: './repositories.gradle'
   buildscript.repositories.addAll(project.repositories)
   dependencies {
@@ -53,10 +59,13 @@ project.ext.externalDependency = [
     'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.10',
     'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.1',
     'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.8',
+    'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.0',
+    'awsRds':'software.amazon.awssdk:rds:2.18.24',
     'cacheApi' : 'javax.cache:cache-api:1.1.0',
     'commonsCli': 'commons-cli:commons-cli:1.5.0',
     'commonsIo': 'commons-io:commons-io:2.4',
     'commonsLang': 'commons-lang:commons-lang:2.6',
+    'commonsText': 'org.apache.commons:commons-text:1.10.0',
     'commonsCollections': 'commons-collections:commons-collections:3.2.2',
     'data' : 'com.linkedin.pegasus:data:' + pegasusVersion,
     'datastaxOssNativeProtocol': 'com.datastax.oss:native-protocol:1.5.1',
@@ -75,23 +84,29 @@ project.ext.externalDependency = [
     'gson': 'com.google.code.gson:gson:2.8.9',
     'guice': 'com.google.inject:guice:4.2.2',
     'guava': 'com.google.guava:guava:27.0.1-jre',
-    'h2': 'com.h2database:h2:2.1.210',
+    'h2': 'com.h2database:h2:2.1.214',
     'hadoopClient': 'org.apache.hadoop:hadoop-client:3.2.1',
     'hadoopCommon':'org.apache.hadoop:hadoop-common:2.7.2',
     'hadoopMapreduceClient':'org.apache.hadoop:hadoop-mapreduce-client-core:2.7.2',
+    'hadoopCommon3':'org.apache.hadoop:hadoop-common:3.3.4',
     'hibernateCore': 'org.hibernate:hibernate-core:5.2.16.Final',
     'httpClient': 'org.apache.httpcomponents:httpclient:4.5.9',
     'httpAsyncClient': 'org.apache.httpcomponents:httpasyncclient:4.1.5',
     'iStackCommons': 'com.sun.istack:istack-commons-runtime:4.0.1',
-    'jacksonCore': 'com.fasterxml.jackson.core:jackson-core:2.13.2',
-    'jacksonDataBind': 'com.fasterxml.jackson.core:jackson-databind:2.13.2.2',
-    'jacksonDataFormatYaml': 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.2',
+    'jacksonJDK8': "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$jacksonVersion",
+    'jacksonDataPropertyFormat': "com.fasterxml.jackson.dataformat:jackson-dataformat-properties:$jacksonVersion",
+    'jacksonCore': "com.fasterxml.jackson.core:jackson-core:$jacksonVersion",
+    'jacksonDataBind': "com.fasterxml.jackson.core:jackson-databind:$jacksonVersion.2",
+    'jacksonDataFormatYaml': "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:$jacksonVersion",
+    'woodstoxCore': 'com.fasterxml.woodstox:woodstox-core:6.4.0',
     'javatuples': 'org.javatuples:javatuples:1.2',
     'javaxInject' : 'javax.inject:javax.inject:1',
     'javaxValidation' : 'javax.validation:validation-api:2.0.1.Final',
     'jerseyCore': 'org.glassfish.jersey.core:jersey-client:2.25.1',
     'jerseyGuava': 'org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1',
-    'jettyJaas': 'org.eclipse.jetty:jetty-jaas:9.4.46.v20220331',
+    'jettyJaas': "org.eclipse.jetty:jetty-jaas:$jettyVersion",
+    'jettyClient': "org.eclipse.jetty:jetty-client:$jettyVersion",
+    'jettison': 'org.codehaus.jettison:jettison:1.5.2',
     'jgrapht': 'org.jgrapht:jgrapht-core:1.5.1',
     'jna': 'net.java.dev.jna:jna:5.12.1',
     'jsonPatch': 'com.github.java-json-tools:json-patch:1.13',
@@ -105,7 +120,11 @@ project.ext.externalDependency = [
     'kafkaAvroSerde': 'io.confluent:kafka-streams-avro-serde:5.5.1',
     'kafkaAvroSerializer': 'io.confluent:kafka-avro-serializer:5.1.4',
     'kafkaClients': 'org.apache.kafka:kafka-clients:2.3.0',
-    'logbackClassic': 'ch.qos.logback:logback-classic:1.2.9',
+    'logbackClassic': "ch.qos.logback:logback-classic:$logbackClassic",
+    'slf4jApi': "org.slf4j:slf4j-api:$slf4jVersion",
+    'log4jCore': "org.apache.logging.log4j:log4j-core:$log4jVersion",
+    'log4jApi': "org.apache.logging.log4j:log4j-api:$log4jVersion",
+    'log4j12Api': "org.slf4j:log4j-over-slf4j:$slf4jVersion",
     'lombok': 'org.projectlombok:lombok:1.18.12',
     'mariadbConnector': 'org.mariadb.jdbc:mariadb-java-client:2.6.0',
     'mavenArtifact': "org.apache.maven:maven-artifact:$mavenVersion",
@@ -120,7 +139,7 @@ project.ext.externalDependency = [
     'opentelemetryApi': 'io.opentelemetry:opentelemetry-api:' + openTelemetryVersion,
     'opentelemetryAnnotations': 'io.opentelemetry:opentelemetry-extension-annotations:' + openTelemetryVersion,
     'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15',
-    'parquet': 'org.apache.parquet:parquet-avro:1.12.2',
+    'parquet': 'org.apache.parquet:parquet-avro:1.12.3',
     'picocli': 'info.picocli:picocli:4.5.0',
     'playCache': 'com.typesafe.play:play-cache_2.12:2.7.6',
     'playEhcache': 'com.typesafe.play:play-ehcache_2.12:2.7.6',
@@ -133,13 +152,15 @@ project.ext.externalDependency = [
     'playTest': 'com.typesafe.play:play-test_2.12:2.7.6',
     'pac4j': 'org.pac4j:pac4j-oidc:3.6.0',
     'playPac4j': 'org.pac4j:play-pac4j_2.12:8.0.2',
-    'postgresql': 'org.postgresql:postgresql:42.3.3',
-    'protobuf': 'com.google.protobuf:protobuf-java:3.19.3',
+    'postgresql': 'org.postgresql:postgresql:42.3.8',
+    'protobuf': 'com.google.protobuf:protobuf-java:3.19.6',
+    'rangerCommons': 'org.apache.ranger:ranger-plugins-common:2.3.0',
     'reflections': 'org.reflections:reflections:0.9.9',
     'resilience4j': 'io.github.resilience4j:resilience4j-retry:1.7.1',
     'rythmEngine': 'org.rythmengine:rythm-engine:1.3.0',
     'servletApi': 'javax.servlet:javax.servlet-api:3.1.0',
-    'shiroCore': 'org.apache.shiro:shiro-core:1.8.0',
+    'shiroCore': 'org.apache.shiro:shiro-core:1.10.0',
+    'snakeYaml': 'org.yaml:snakeyaml:1.33',
     'sparkSql' : 'org.apache.spark:spark-sql_2.11:2.4.8',
     'sparkHive' : 'org.apache.spark:spark-hive_2.11:2.4.8',
     'springBeans': "org.springframework:spring-beans:$springVersion",
@@ -176,11 +197,18 @@ allprojects {
   apply plugin: 'checkstyle'
 }
 
-configure(subprojects.findAll {! it.name.startsWith('spark-lineage') }) {
+configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
 
   configurations.all {
     exclude group: "io.netty", module: "netty"
     exclude group: "log4j", module: "log4j"
+    exclude group: "org.springframework.boot", module: "spring-boot-starter-logging"
+    exclude group: "org.apache.logging.log4j", module: "log4j-to-slf4j"
+    exclude group: "com.vaadin.external.google", module: "android-json"
+    exclude group: "org.slf4j", module: "slf4j-reload4j"
+    exclude group: "org.slf4j", module: "slf4j-log4j12"
+    exclude group: "org.slf4j", module: "slf4j-nop"
+    exclude group: "org.slf4j", module: "slf4j-ext"
   }
 }
 
@@ -206,8 +234,8 @@ subprojects {
         implementation('org.apache.commons:commons-compress:1.21')
         implementation('org.apache.velocity:velocity-engine-core:2.3')
         implementation('org.hibernate:hibernate-validator:6.0.20.Final')
-        implementation('com.fasterxml.jackson.core:jackson-databind:2.13.2.2')
-        implementation('com.fasterxml.jackson.core:jackson-dataformat-cbor:2.13.2')
+        implementation("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion.2")
+        implementation("com.fasterxml.jackson.core:jackson-dataformat-cbor:$jacksonVersion")
       }
     }
 

buildSrc/build.gradle

@@ -10,7 +10,7 @@ dependencies {
     exclude group: 'com.google.guava', module: 'guava'
   }
   compile 'com.google.guava:guava:27.0.1-jre'
-  compile 'com.fasterxml.jackson.core:jackson-databind:2.9.10.7'
-  compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.8.11'
+  compile 'com.fasterxml.jackson.core:jackson-databind:2.13.4.2'
+  compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.4'
   compile 'commons-io:commons-io:2.11.0'
 }

datahub-frontend/app/client/AuthServiceClient.java

@@ -3,6 +3,8 @@
 import com.datahub.authentication.Authentication;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import java.nio.charset.StandardCharsets;
 import java.util.Objects;
 import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
@@ -150,9 +152,11 @@ public boolean signUp(@Nonnull final String userUrn, @Nonnull final String fullN
         final String jsonStr = EntityUtils.toString(entity);
         return getIsNativeUserCreatedFromJson(jsonStr);
       } else {
+        String content = response.getEntity().getContent() == null ? "" : new String(
+                response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
         throw new RuntimeException(
-            String.format("Bad response from the Metadata Service: %s %s", response.getStatusLine().toString(),
-                response.getEntity().toString()));
+            String.format("Bad response from the Metadata Service: %s %s Body: %s", response.getStatusLine().toString(),
+                response.getEntity().toString(), content));
       }
     } catch (Exception e) {
       throw new RuntimeException(String.format("Failed to create user %s", userUrn), e);

datahub-frontend/app/security/AuthenticationManager.java

@@ -51,15 +51,15 @@ public void handle(@Nonnull Callback[] callbacks) {
       NameCallback nc = null;
       PasswordCallback pc = null;
       for (Callback callback : callbacks) {
-        Logger.error("The submitted callback is of type: " + callback.getClass() + " : " + callback);
+        Logger.debug("The submitted callback is of type: " + callback.getClass() + " : " + callback);
         if (callback instanceof NameCallback) {
           nc = (NameCallback) callback;
           nc.setName(this.username);
         } else if (callback instanceof PasswordCallback) {
           pc = (PasswordCallback) callback;
           pc.setPassword(this.password.toCharArray());
         } else {
-          Logger.warn("The submitted callback is unsupported! ", callback);
+          Logger.warn("The submitted callback is unsupported! type: " + callback.getClass(), callback);
         }
       }
     }

datahub-frontend/play.gradle

@@ -17,7 +17,7 @@ dependencies {
 
   constraints {
     play('org.springframework:spring-core:5.2.3.RELEASE')
-    play('com.fasterxml.jackson.core:jackson-databind:2.9.10.4')
+    play(externalDependency.jacksonDataBind)
     play('com.nimbusds:nimbus-jose-jwt:7.9')
     play('com.typesafe.akka:akka-actor_2.12:2.5.16')
     play('net.minidev:json-smart:2.4.1')
@@ -55,13 +55,14 @@ dependencies {
   testImplementation externalDependency.playTest
   testCompile externalDependency.testng
 
+  implementation externalDependency.slf4jApi
   compileOnly externalDependency.lombok
   runtime externalDependency.guice
   runtime (externalDependency.playDocs) {
     exclude group: 'com.typesafe.akka', module: 'akka-http-core_2.12'
   }
   runtime externalDependency.playGuice
-  runtime externalDependency.logbackClassic
+  implementation externalDependency.logbackClassic
 
   annotationProcessor externalDependency.lombok
 }

datahub-graphql-core/build.gradle

@@ -6,6 +6,7 @@ apply plugin: 'java'
 dependencies {
     compile project(':metadata-service:restli-client')
     compile project(':metadata-service:auth-impl')
+    compile project(':metadata-service:auth-api')
     compile project(':metadata-io')
     compile project(':metadata-utils')
 
@@ -15,6 +16,7 @@ dependencies {
     compile externalDependency.antlr4
     compile externalDependency.guava
 
+    implementation externalDependency.slf4jApi
     compileOnly externalDependency.lombok
     annotationProcessor externalDependency.lombok
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -3,11 +3,11 @@
 import com.datahub.authentication.AuthenticationConfiguration;
 import com.datahub.authentication.group.GroupService;
 import com.datahub.authentication.invite.InviteTokenService;
-import com.datahub.authentication.post.PostService;
 import com.datahub.authentication.token.StatefulTokenService;
 import com.datahub.authentication.user.NativeUserService;
 import com.datahub.authorization.AuthorizationConfiguration;
 import com.datahub.authorization.role.RoleService;
+import com.datahub.authentication.post.PostService;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.VersionedUrn;
 import com.linkedin.common.urn.Urn;
@@ -249,7 +249,7 @@
 import com.linkedin.datahub.graphql.types.tag.TagType;
 import com.linkedin.datahub.graphql.types.test.TestType;
 import com.linkedin.entity.client.EntityClient;
-import com.linkedin.metadata.config.DatahubConfiguration;
+import com.linkedin.metadata.config.DataHubConfiguration;
 import com.linkedin.metadata.config.IngestionConfiguration;
 import com.linkedin.metadata.config.TestsConfiguration;
 import com.linkedin.metadata.config.VisualConfiguration;
@@ -327,7 +327,7 @@ public class GmsGraphQLEngine {
     private final VisualConfiguration visualConfiguration;
     private final TelemetryConfiguration telemetryConfiguration;
     private final TestsConfiguration testsConfiguration;
-    private final DatahubConfiguration datahubConfiguration;
+    private final DataHubConfiguration datahubConfiguration;
 
     private final DatasetType datasetType;
     private final CorpUserType corpUserType;
@@ -393,7 +393,7 @@ public GmsGraphQLEngine(final EntityClient entityClient, final GraphClient graph
         final AuthorizationConfiguration authorizationConfiguration, final GitVersion gitVersion,
         final TimelineService timelineService, final boolean supportsImpactAnalysis,
         final VisualConfiguration visualConfiguration, final TelemetryConfiguration telemetryConfiguration,
-        final TestsConfiguration testsConfiguration, final DatahubConfiguration datahubConfiguration,
+        final TestsConfiguration testsConfiguration, final DataHubConfiguration datahubConfiguration,
         final SiblingGraphService siblingGraphService, final GroupService groupService, final RoleService roleService,
         final InviteTokenService inviteTokenService, final PostService postService, final FeatureFlags featureFlags) {
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/QueryContext.java

@@ -2,7 +2,7 @@
 
 import com.datahub.authentication.Actor;
 import com.datahub.authentication.Authentication;
-import com.datahub.authorization.Authorizer;
+import com.datahub.plugins.auth.authorization.Authorizer;
 
 
 /**
@@ -16,7 +16,7 @@ public interface QueryContext {
     boolean isAuthenticated();
 
     /**
-     * Returns the {@link com.datahub.authentication.Authentication} associated with the current query context.
+     * Returns the {@link Authentication} associated with the current query context.
      */
     Authentication getAuthentication();
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java

@@ -2,7 +2,7 @@
 
 import com.datahub.authorization.AuthorizationRequest;
 import com.datahub.authorization.AuthorizationResult;
-import com.datahub.authorization.Authorizer;
+import com.datahub.plugins.auth.authorization.Authorizer;
 import com.datahub.authorization.ResourceSpec;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.AuditStamp;

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/AuthUtils.java

@@ -7,7 +7,7 @@
 import java.util.Optional;
 import com.datahub.authorization.AuthorizationRequest;
 import com.datahub.authorization.AuthorizationResult;
-import com.datahub.authorization.Authorizer;
+import com.datahub.plugins.auth.authorization.Authorizer;
 
 public class AuthUtils {
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java

@@ -2,7 +2,7 @@
 
 import com.datahub.authorization.AuthorizationRequest;
 import com.datahub.authorization.AuthorizationResult;
-import com.datahub.authorization.Authorizer;
+import com.datahub.plugins.auth.authorization.Authorizer;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java

@@ -16,7 +16,7 @@
 import com.linkedin.datahub.graphql.generated.TelemetryConfig;
 import com.linkedin.datahub.graphql.generated.TestsConfig;
 import com.linkedin.datahub.graphql.generated.VisualConfig;
-import com.linkedin.metadata.config.DatahubConfiguration;
+import com.linkedin.metadata.config.DataHubConfiguration;
 import com.linkedin.metadata.config.IngestionConfiguration;
 import com.linkedin.metadata.config.TestsConfiguration;
 import com.linkedin.metadata.telemetry.TelemetryConfiguration;
@@ -42,7 +42,7 @@ public class AppConfigResolver implements DataFetcher<CompletableFuture<AppConfi
   private final VisualConfiguration _visualConfiguration;
   private final TelemetryConfiguration _telemetryConfiguration;
   private final TestsConfiguration _testsConfiguration;
-  private final DatahubConfiguration _datahubConfiguration;
+  private final DataHubConfiguration _datahubConfiguration;
 
   public AppConfigResolver(
       final GitVersion gitVersion,
@@ -54,7 +54,7 @@ public AppConfigResolver(
       final VisualConfiguration visualConfiguration,
       final TelemetryConfiguration telemetryConfiguration,
       final TestsConfiguration testsConfiguration,
-      final DatahubConfiguration datahubConfiguration) {
+      final DataHubConfiguration datahubConfiguration) {
     _gitVersion = gitVersion;
     _isAnalyticsEnabled = isAnalyticsEnabled;
     _ingestionConfiguration = ingestionConfiguration;

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionAuthUtils.java

@@ -1,6 +1,6 @@
 package com.linkedin.datahub.graphql.resolvers.ingest;
 
-import com.datahub.authorization.Authorizer;
+import com.datahub.plugins.auth.authorization.Authorizer;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.metadata.authorization.PoliciesConfig;