Merge branch 'master' into s3-config-pathspec

.github/pr-labeler-config.yml

@@ -9,6 +9,7 @@ product:
 - 'datahub-web-react/**/*'
 - 'datahub-frontend/**/*'
 - 'datahub-graphql-core/**/*'
+- 'metadata-io/**/*'
 
 docs:
 - 'docs/**/*'

.github/workflows/build-and-test.yml

@@ -25,10 +25,12 @@ jobs:
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+        with:
+          fetch-depth: 0
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/check-datahub-jars.yml

@@ -30,10 +30,10 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/docker-unified.yml

@@ -359,10 +359,10 @@ jobs:
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/documentation.yml

@@ -19,10 +19,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.10"

.github/workflows/metadata-io.yml

@@ -29,10 +29,10 @@ jobs:
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/publish-datahub-jars.yml

@@ -53,10 +53,10 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/spark-smoke-test.yml

@@ -28,10 +28,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.gitignore

@@ -37,7 +37,7 @@ MANIFEST
 **/build
 /config
 */i18n
-/out
+out/
 
 # Mac OS
 **/.DS_Store
@@ -68,7 +68,7 @@ metadata-ingestion/generated/**
 # docs
 docs/generated/
 tmp*
-temp*
+temp/**
 
 # frontend assets
 datahub-frontend/public/**

README.md

@@ -150,7 +150,7 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 - [Data Catalogue — Knowing your data](https://medium.com/albert-franzi/data-catalogue-knowing-your-data-15f7d0724900)
 - [DataHub: A Generalized Metadata Search & Discovery Tool](https://engineering.linkedin.com/blog/2019/data-hub)
 - [Open sourcing DataHub: LinkedIn’s metadata search and discovery platform](https://engineering.linkedin.com/blog/2020/open-sourcing-datahub--linkedins-metadata-search-and-discovery-p)
-- [Emerging Architectures for Modern Data Infrastructure](https://a16z.com/2020/10/15/the-emerging-architectures-for-modern-data-infrastructure/)
+- [Emerging Architectures for Modern Data Infrastructure](https://future.com/emerging-architectures-for-modern-data-infrastructure-2020/)
 
 See the full list [here](docs/links.md).
 

SECURITY.md

@@ -0,0 +1,18 @@
+# Reporting security issues
+
+If you think you have found a security vulnerability, please send a report to [email protected]. This address can be used for all of Acryl Data’s open source and commercial products (including but not limited to DataHub and Acryl Data). We can accept only vulnerability reports at this address.
+
+It's not mandatory, but if you'd like to encrypt your message to us; please use our PGP key. The key fingerprint is:
+
+A50B10A86CC21F4B7BE102E170764C95B4FACEBF
+
+The key is available from [keyserver.ubuntu.com](https://keyserver.ubuntu.com/pks/lookup?search=A50B10A86CC21F4B7BE102E170764C95B4FACEBF&fingerprint=on&op=index).
+
+Acryl Data will send you a response indicating the next steps in handling your report. After the initial reply to your report, the security team will keep you informed of the progress towards a fix and full announcement, and may ask for additional information or guidance.
+
+**Important:** We ask you to not disclose the vulnerability before it have been fixed and announced, unless you received a response from the Acryl Data security team that you can do so.
+
+## Security announcements
+
+We maintain [Security Advisories](https://github.com/datahub-project/datahub/security/advisories) on the DataHub project GitHub repository,
+where we will post a summary, remediation, and mitigation details for any patch containing security fixes.

build.gradle

@@ -4,16 +4,15 @@ buildscript {
   ext.mavenVersion = '3.6.3'
   ext.springVersion = '5.3.20'
   ext.springBootVersion = '2.5.12'
+  ext.neo4jVersion = '4.4.9'
+  ext.graphQLJavaVersion = '19.0'
   apply from: './repositories.gradle'
   buildscript.repositories.addAll(project.repositories)
   dependencies {
     classpath 'com.linkedin.pegasus:gradle-plugins:' + pegasusVersion
     classpath 'com.github.node-gradle:gradle-node-plugin:2.2.4'
     classpath 'com.commercehub.gradle.plugin:gradle-avro-plugin:0.8.1'
     classpath 'org.springframework.boot:spring-boot-gradle-plugin:' + springBootVersion
-    classpath('com.github.jengelman.gradle.plugins:shadow:5.2.0') {
-      exclude group: 'org.apache.logging.log4j', module: 'log4j-core'
-    }
     classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0"
     classpath "com.palantir.gradle.gitversion:gradle-git-version:0.12.3"
     classpath "org.gradle.playframework:gradle-playframework:0.12"
@@ -23,6 +22,7 @@ buildscript {
 
 plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.0-rc2'
+  id 'com.github.johnrengelman.shadow' version '6.1.0'
 }
 
 project.ext.spec = [
@@ -50,6 +50,7 @@ project.ext.externalDependency = [
     'avroCompiler_1_7': 'org.apache.avro:avro-compiler:1.7.7',
     'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.10',
     'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.1',
+    'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.8',
     'cacheApi' : 'javax.cache:cache-api:1.1.0',
     'commonsCli': 'commons-cli:commons-cli:1.5.0',
     'commonsIo': 'commons-io:commons-io:2.4',
@@ -67,7 +68,8 @@ project.ext.externalDependency = [
     'elasticSearchRest': 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.9.3',
     'elasticSearchTransport': 'org.elasticsearch.client:transport:7.9.3',
     'findbugsAnnotations': 'com.google.code.findbugs:annotations:3.0.1',
-    'graphqlJava': 'com.graphql-java:graphql-java:16.1',
+    'graphqlJava': 'com.graphql-java:graphql-java:' + graphQLJavaVersion,
+    'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:' + graphQLJavaVersion,
     'gson': 'com.google.code.gson:gson:2.8.9',
     'guice': 'com.google.inject:guice:4.2.2',
     'guava': 'com.google.guava:guava:27.0.1-jre',
@@ -89,6 +91,7 @@ project.ext.externalDependency = [
     'jerseyGuava': 'org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1',
     'jettyJaas': 'org.eclipse.jetty:jetty-jaas:9.4.46.v20220331',
     'jgrapht': 'org.jgrapht:jgrapht-core:1.5.1',
+    'jsonPatch': 'com.github.java-json-tools:json-patch:1.13',
     'jsonSchemaAvro': 'com.github.fge:json-schema-avro:0.1.4',
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'jsonSmart': 'net.minidev:json-smart:2.4.6',
@@ -109,8 +112,8 @@ project.ext.externalDependency = [
     'mockServer': 'org.mock-server:mockserver-netty:5.11.2',
     'mockServerClient': 'org.mock-server:mockserver-client-java:5.11.2',
     'mysqlConnector': 'mysql:mysql-connector-java:8.0.20',
-    'neo4jHarness': 'org.neo4j.test:neo4j-harness:3.4.11',
-    'neo4jJavaDriver': 'org.neo4j.driver:neo4j-java-driver:4.0.1',
+    'neo4jHarness': 'org.neo4j.test:neo4j-harness:' + neo4jVersion,
+    'neo4jJavaDriver': 'org.neo4j.driver:neo4j-java-driver:' + neo4jVersion,
     'opentelemetryApi': 'io.opentelemetry:opentelemetry-api:1.0.0',
     'opentelemetryAnnotations': 'io.opentelemetry:opentelemetry-extension-annotations:1.0.0',
     'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15',
@@ -169,7 +172,7 @@ allprojects {
   apply plugin: 'checkstyle'
 }
 
-configure(subprojects.findAll {it.name != 'spark-lineage'}) {
+configure(subprojects.findAll {! it.name.startsWith('spark-lineage') }) {
 
   configurations.all {
     exclude group: "io.netty", module: "netty"
@@ -219,37 +222,14 @@ subprojects {
     }
   }
 
-  if (project.name != 'datahub-protobuf') {
-    tasks.withType(JavaCompile).configureEach {
-      javaCompiler = javaToolchains.compilerFor {
-        languageVersion = JavaLanguageVersion.of(8)
-      }
-    }
-    tasks.withType(Test).configureEach {
-      javaLauncher = javaToolchains.launcherFor {
-        languageVersion = JavaLanguageVersion.of(8)
-      }
-    }
-  } else {
-    tasks.withType(JavaExec).configureEach {
-      javaLauncher = javaToolchains.launcherFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
+  tasks.withType(JavaCompile).configureEach {
+    javaCompiler = javaToolchains.compilerFor {
+      languageVersion = JavaLanguageVersion.of(11)
     }
-    tasks.withType(Javadoc).configureEach {
-    javadocTool = javaToolchains.javadocToolFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
-    }
-    tasks.withType(JavaCompile).configureEach {
-      javaCompiler = javaToolchains.compilerFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
-    }
-    tasks.withType(Test).configureEach {
-      javaLauncher = javaToolchains.launcherFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
+  }
+  tasks.withType(Test).configureEach {
+    javaLauncher = javaToolchains.launcherFor {
+      languageVersion = JavaLanguageVersion.of(11)
     }
   }
 

datahub-frontend/app/client/AuthServiceClient.java

@@ -1,6 +1,8 @@
 package client;
 
+import com.datahub.authentication.Authentication;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import java.util.Objects;
 import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
@@ -13,7 +15,6 @@
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.util.EntityUtils;
 import play.mvc.Http;
-import com.datahub.authentication.Authentication;
 
 
 /**
@@ -66,11 +67,15 @@ public String generateSessionTokenForUser(@Nonnull final String userId) {
     try {
 
       final String protocol = this.metadataServiceUseSsl ? "https" : "http";
-      final HttpPost request = new HttpPost(String.format("%s://%s:%s/%s", protocol, this.metadataServiceHost,
-          this.metadataServicePort, GENERATE_SESSION_TOKEN_ENDPOINT));
+      final HttpPost request = new HttpPost(
+          String.format("%s://%s:%s/%s", protocol, this.metadataServiceHost, this.metadataServicePort,
+              GENERATE_SESSION_TOKEN_ENDPOINT));
 
       // Build JSON request to generate a token on behalf of a user.
-      String json = String.format("{ \"%s\":\"%s\" }", USER_ID_FIELD, userId);
+      final ObjectMapper objectMapper = new ObjectMapper();
+      final ObjectNode objectNode = objectMapper.createObjectNode();
+      objectNode.put(USER_ID_FIELD, userId);
+      final String json = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(objectNode);
       request.setEntity(new StringEntity(json));
 
       // Add authorization header with DataHub frontend system id and secret.
@@ -101,7 +106,6 @@ public String generateSessionTokenForUser(@Nonnull final String userId) {
   /**
    * Call the Auth Service to create a native Datahub user.
    */
-  @Nonnull
   public boolean signUp(@Nonnull final String userUrn, @Nonnull final String fullName, @Nonnull final String email,
       @Nonnull final String title, @Nonnull final String password, @Nonnull final String inviteToken) {
     Objects.requireNonNull(userUrn, "userUrn must not be null");
@@ -115,15 +119,20 @@ public boolean signUp(@Nonnull final String userUrn, @Nonnull final String fullN
     try {
 
       final String protocol = this.metadataServiceUseSsl ? "https" : "http";
-      final HttpPost request =
-          new HttpPost(String.format("%s://%s:%s/%s", protocol, this.metadataServiceHost, this.metadataServicePort,
+      final HttpPost request = new HttpPost(
+          String.format("%s://%s:%s/%s", protocol, this.metadataServiceHost, this.metadataServicePort,
               SIGN_UP_ENDPOINT));
 
-      // Build JSON request to verify credentials for a native user.
-      String json =
-          String.format("{ \"%s\":\"%s\", \"%s\":\"%s\", \"%s\":\"%s\", \"%s\":\"%s\", \"%s\":\"%s\", \"%s\":\"%s\" }",
-              USER_URN_FIELD, userUrn, FULL_NAME_FIELD, fullName, EMAIL_FIELD, email, TITLE_FIELD, title,
-              PASSWORD_FIELD, password, INVITE_TOKEN_FIELD, inviteToken);
+      // Build JSON request to sign up a native user.
+      final ObjectMapper objectMapper = new ObjectMapper();
+      final ObjectNode objectNode = objectMapper.createObjectNode();
+      objectNode.put(USER_URN_FIELD, userUrn);
+      objectNode.put(FULL_NAME_FIELD, fullName);
+      objectNode.put(EMAIL_FIELD, email);
+      objectNode.put(TITLE_FIELD, title);
+      objectNode.put(PASSWORD_FIELD, password);
+      objectNode.put(INVITE_TOKEN_FIELD, inviteToken);
+      final String json = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(objectNode);
       request.setEntity(new StringEntity(json));
 
       // Add authorization header with DataHub frontend system id and secret.
@@ -141,7 +150,7 @@ public boolean signUp(@Nonnull final String userUrn, @Nonnull final String fullN
                 response.getEntity().toString()));
       }
     } catch (Exception e) {
-      throw new RuntimeException("Failed to create user", e);
+      throw new RuntimeException(String.format("Failed to create user %s", userUrn), e);
     } finally {
       try {
         httpClient.close();
@@ -154,7 +163,6 @@ public boolean signUp(@Nonnull final String userUrn, @Nonnull final String fullN
   /**
    * Call the Auth Service to reset credentials for a native DataHub user.
    */
-  @Nonnull
   public boolean resetNativeUserCredentials(@Nonnull final String userUrn, @Nonnull final String password,
       @Nonnull final String resetToken) {
     Objects.requireNonNull(userUrn, "userUrn must not be null");
@@ -170,9 +178,12 @@ public boolean resetNativeUserCredentials(@Nonnull final String userUrn, @Nonnul
               RESET_NATIVE_USER_CREDENTIALS_ENDPOINT));
 
       // Build JSON request to verify credentials for a native user.
-      String json =
-          String.format("{ \"%s\":\"%s\", \"%s\":\"%s\", \"%s\":\"%s\" }", USER_URN_FIELD, userUrn,
-              PASSWORD_FIELD, password, RESET_TOKEN_FIELD, resetToken);
+      final ObjectMapper objectMapper = new ObjectMapper();
+      final ObjectNode objectNode = objectMapper.createObjectNode();
+      objectNode.put(USER_URN_FIELD, userUrn);
+      objectNode.put(PASSWORD_FIELD, password);
+      objectNode.put(RESET_TOKEN_FIELD, resetToken);
+      final String json = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(objectNode);
       request.setEntity(new StringEntity(json));
 
       // Add authorization header with DataHub frontend system id and secret.
@@ -203,7 +214,6 @@ public boolean resetNativeUserCredentials(@Nonnull final String userUrn, @Nonnul
   /**
    * Call the Auth Service to verify the credentials for a native Datahub user.
    */
-  @Nonnull
   public boolean verifyNativeUserCredentials(@Nonnull final String userUrn, @Nonnull final String password) {
     Objects.requireNonNull(userUrn, "userUrn must not be null");
     Objects.requireNonNull(password, "password must not be null");
@@ -217,8 +227,11 @@ public boolean verifyNativeUserCredentials(@Nonnull final String userUrn, @Nonnu
               VERIFY_NATIVE_USER_CREDENTIALS_ENDPOINT));
 
       // Build JSON request to verify credentials for a native user.
-      String json =
-          String.format("{ \"%s\":\"%s\", \"%s\":\"%s\" }", USER_URN_FIELD, userUrn, PASSWORD_FIELD, password);
+      final ObjectMapper objectMapper = new ObjectMapper();
+      final ObjectNode objectNode = objectMapper.createObjectNode();
+      objectNode.put(USER_URN_FIELD, userUrn);
+      objectNode.put(PASSWORD_FIELD, password);
+      final String json = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(objectNode);
       request.setEntity(new StringEntity(json));
 
       // Add authorization header with DataHub frontend system id and secret.