Merge branch 'master' into ttaubermarshall-corp-user-entity

.github/pr-labeler-config.yml

@@ -9,6 +9,7 @@ product:
 - 'datahub-web-react/**/*'
 - 'datahub-frontend/**/*'
 - 'datahub-graphql-core/**/*'
+- 'metadata-io/**/*'
 
 docs:
 - 'docs/**/*'

.github/workflows/build-and-test.yml

@@ -25,10 +25,12 @@ jobs:
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+        with:
+          fetch-depth: 0
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/check-datahub-jars.yml

@@ -30,10 +30,10 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/docker-unified.yml

@@ -359,10 +359,10 @@ jobs:
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/documentation.yml

@@ -19,10 +19,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.10"

.github/workflows/metadata-io.yml

@@ -29,10 +29,10 @@ jobs:
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/publish-datahub-jars.yml

@@ -53,10 +53,10 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/spark-smoke-test.yml

@@ -28,10 +28,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.gitignore

@@ -37,7 +37,7 @@ MANIFEST
 **/build
 /config
 */i18n
-/out
+out/
 
 # Mac OS
 **/.DS_Store

README.md

@@ -150,7 +150,7 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 - [Data Catalogue — Knowing your data](https://medium.com/albert-franzi/data-catalogue-knowing-your-data-15f7d0724900)
 - [DataHub: A Generalized Metadata Search & Discovery Tool](https://engineering.linkedin.com/blog/2019/data-hub)
 - [Open sourcing DataHub: LinkedIn’s metadata search and discovery platform](https://engineering.linkedin.com/blog/2020/open-sourcing-datahub--linkedins-metadata-search-and-discovery-p)
-- [Emerging Architectures for Modern Data Infrastructure](https://a16z.com/2020/10/15/the-emerging-architectures-for-modern-data-infrastructure/)
+- [Emerging Architectures for Modern Data Infrastructure](https://future.com/emerging-architectures-for-modern-data-infrastructure-2020/)
 
 See the full list [here](docs/links.md).
 

SECURITY.md

@@ -0,0 +1,18 @@
+# Reporting security issues
+
+If you think you have found a security vulnerability, please send a report to [email protected]. This address can be used for all of Acryl Data’s open source and commercial products (including but not limited to DataHub and Acryl Data). We can accept only vulnerability reports at this address.
+
+It's not mandatory, but if you'd like to encrypt your message to us; please use our PGP key. The key fingerprint is:
+
+A50B10A86CC21F4B7BE102E170764C95B4FACEBF
+
+The key is available from [keyserver.ubuntu.com](https://keyserver.ubuntu.com/pks/lookup?search=A50B10A86CC21F4B7BE102E170764C95B4FACEBF&fingerprint=on&op=index).
+
+Acryl Data will send you a response indicating the next steps in handling your report. After the initial reply to your report, the security team will keep you informed of the progress towards a fix and full announcement, and may ask for additional information or guidance.
+
+**Important:** We ask you to not disclose the vulnerability before it have been fixed and announced, unless you received a response from the Acryl Data security team that you can do so.
+
+## Security announcements
+
+We maintain [Security Advisories](https://github.com/datahub-project/datahub/security/advisories) on the DataHub project GitHub repository,
+where we will post a summary, remediation, and mitigation details for any patch containing security fixes.

build.gradle

@@ -4,16 +4,15 @@ buildscript {
   ext.mavenVersion = '3.6.3'
   ext.springVersion = '5.3.20'
   ext.springBootVersion = '2.5.12'
+  ext.neo4jVersion = '4.4.9'
+  ext.graphQLJavaVersion = '19.0'
   apply from: './repositories.gradle'
   buildscript.repositories.addAll(project.repositories)
   dependencies {
     classpath 'com.linkedin.pegasus:gradle-plugins:' + pegasusVersion
     classpath 'com.github.node-gradle:gradle-node-plugin:2.2.4'
     classpath 'com.commercehub.gradle.plugin:gradle-avro-plugin:0.8.1'
     classpath 'org.springframework.boot:spring-boot-gradle-plugin:' + springBootVersion
-    classpath('com.github.jengelman.gradle.plugins:shadow:5.2.0') {
-      exclude group: 'org.apache.logging.log4j', module: 'log4j-core'
-    }
     classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0"
     classpath "com.palantir.gradle.gitversion:gradle-git-version:0.12.3"
     classpath "org.gradle.playframework:gradle-playframework:0.12"
@@ -23,6 +22,7 @@ buildscript {
 
 plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.0-rc2'
+  id 'com.github.johnrengelman.shadow' version '6.1.0'
 }
 
 project.ext.spec = [
@@ -68,7 +68,8 @@ project.ext.externalDependency = [
     'elasticSearchRest': 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.9.3',
     'elasticSearchTransport': 'org.elasticsearch.client:transport:7.9.3',
     'findbugsAnnotations': 'com.google.code.findbugs:annotations:3.0.1',
-    'graphqlJava': 'com.graphql-java:graphql-java:16.1',
+    'graphqlJava': 'com.graphql-java:graphql-java:' + graphQLJavaVersion,
+    'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:' + graphQLJavaVersion,
     'gson': 'com.google.code.gson:gson:2.8.9',
     'guice': 'com.google.inject:guice:4.2.2',
     'guava': 'com.google.guava:guava:27.0.1-jre',
@@ -111,8 +112,8 @@ project.ext.externalDependency = [
     'mockServer': 'org.mock-server:mockserver-netty:5.11.2',
     'mockServerClient': 'org.mock-server:mockserver-client-java:5.11.2',
     'mysqlConnector': 'mysql:mysql-connector-java:8.0.20',
-    'neo4jHarness': 'org.neo4j.test:neo4j-harness:3.4.11',
-    'neo4jJavaDriver': 'org.neo4j.driver:neo4j-java-driver:4.0.1',
+    'neo4jHarness': 'org.neo4j.test:neo4j-harness:' + neo4jVersion,
+    'neo4jJavaDriver': 'org.neo4j.driver:neo4j-java-driver:' + neo4jVersion,
     'opentelemetryApi': 'io.opentelemetry:opentelemetry-api:1.0.0',
     'opentelemetryAnnotations': 'io.opentelemetry:opentelemetry-extension-annotations:1.0.0',
     'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15',
@@ -171,7 +172,7 @@ allprojects {
   apply plugin: 'checkstyle'
 }
 
-configure(subprojects.findAll {it.name != 'spark-lineage'}) {
+configure(subprojects.findAll {! it.name.startsWith('spark-lineage') }) {
 
   configurations.all {
     exclude group: "io.netty", module: "netty"
@@ -221,37 +222,14 @@ subprojects {
     }
   }
 
-  if (project.name != 'datahub-protobuf') {
-    tasks.withType(JavaCompile).configureEach {
-      javaCompiler = javaToolchains.compilerFor {
-        languageVersion = JavaLanguageVersion.of(8)
-      }
-    }
-    tasks.withType(Test).configureEach {
-      javaLauncher = javaToolchains.launcherFor {
-        languageVersion = JavaLanguageVersion.of(8)
-      }
-    }
-  } else {
-    tasks.withType(JavaExec).configureEach {
-      javaLauncher = javaToolchains.launcherFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
+  tasks.withType(JavaCompile).configureEach {
+    javaCompiler = javaToolchains.compilerFor {
+      languageVersion = JavaLanguageVersion.of(11)
     }
-    tasks.withType(Javadoc).configureEach {
-    javadocTool = javaToolchains.javadocToolFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
-    }
-    tasks.withType(JavaCompile).configureEach {
-      javaCompiler = javaToolchains.compilerFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
-    }
-    tasks.withType(Test).configureEach {
-      javaLauncher = javaToolchains.launcherFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
+  }
+  tasks.withType(Test).configureEach {
+    javaLauncher = javaToolchains.launcherFor {
+      languageVersion = JavaLanguageVersion.of(11)
     }
   }
 

datahub-frontend/app/client/AuthServiceClient.java

@@ -123,7 +123,7 @@ public boolean signUp(@Nonnull final String userUrn, @Nonnull final String fullN
           String.format("%s://%s:%s/%s", protocol, this.metadataServiceHost, this.metadataServicePort,
               SIGN_UP_ENDPOINT));
 
-      // Build JSON request to verify credentials for a native user.
+      // Build JSON request to sign up a native user.
       final ObjectMapper objectMapper = new ObjectMapper();
       final ObjectNode objectNode = objectMapper.createObjectNode();
       objectNode.put(USER_URN_FIELD, userUrn);
@@ -150,7 +150,7 @@ public boolean signUp(@Nonnull final String userUrn, @Nonnull final String fullN
                 response.getEntity().toString()));
       }
     } catch (Exception e) {
-      throw new RuntimeException("Failed to create user", e);
+      throw new RuntimeException(String.format("Failed to create user %s", userUrn), e);
     } finally {
       try {
         httpClient.close();

datahub-frontend/app/controllers/AuthenticationController.java

@@ -150,8 +150,7 @@ public Result logIn(Http.Request request) {
     }
 
     /**
-     * Sign up a native user based on a name, email, title, and password. The invite token must match the global invite
-     * token stored for the DataHub instance.
+     * Sign up a native user based on a name, email, title, and password. The invite token must match an existing invite token.
      *
      */
     @Nonnull
@@ -199,7 +198,7 @@ public Result signUp(Http.Request request) {
 
         final Urn userUrn = new CorpuserUrn(email);
         final String userUrnString = userUrn.toString();
-        boolean isNativeUserCreated = _authClient.signUp(userUrnString, fullName, email, title, password, inviteToken);
+        _authClient.signUp(userUrnString, fullName, email, title, password, inviteToken);
         final String accessToken = _authClient.generateSessionTokenForUser(userUrn.getId());
         return ok().withSession(createSessionMap(userUrnString, accessToken))
             .withCookies(Http.Cookie.builder(ACTOR, userUrnString)
@@ -209,7 +208,7 @@ public Result signUp(Http.Request request) {
     }
 
     /**
-     * Create a native user based on a name, email, and password.
+     * Reset a native user's credentials based on a username, old password, and new password.
      *
      */
     @Nonnull
@@ -245,9 +244,7 @@ public Result resetNativeUserCredentials(Http.Request request) {
 
         final Urn userUrn = new CorpuserUrn(email);
         final String userUrnString = userUrn.toString();
-        boolean areNativeUserCredentialsReset =
-            _authClient.resetNativeUserCredentials(userUrnString, password, resetToken);
-        _logger.debug(String.format("Are native user credentials reset: %b", areNativeUserCredentialsReset));
+        _authClient.resetNativeUserCredentials(userUrnString, password, resetToken);
         final String accessToken = _authClient.generateSessionTokenForUser(userUrn.getId());
         return ok().withSession(createSessionMap(userUrnString, accessToken))
             .withCookies(Http.Cookie.builder(ACTOR, userUrnString)
@@ -288,7 +285,6 @@ private String encodeRedirectUri(final String redirectUri) {
     }
 
     private boolean tryLogin(String username, String password) {
-        JsonNode invalidCredsJson = Json.newObject().put("message", "Invalid Credentials");
         boolean loginSucceeded = false;
 
         // First try jaas login, if enabled

datahub-graphql-core/build.gradle

@@ -9,7 +9,8 @@ dependencies {
     compile project(':metadata-io')
     compile project(':metadata-utils')
 
-    compile externalDependency.graphqlJava
+    implementation externalDependency.graphqlJava
+    implementation externalDependency.graphqlJavaScalars
     compile externalDependency.antlr4Runtime
     compile externalDependency.antlr4
     compile externalDependency.guava