Merge branch 'master' into jj--add-policies-guide

.github/pr-labeler-config.yml

@@ -9,6 +9,7 @@ product:
 - 'datahub-web-react/**/*'
 - 'datahub-frontend/**/*'
 - 'datahub-graphql-core/**/*'
+- 'metadata-io/**/*'
 
 docs:
 - 'docs/**/*'

.github/workflows/build-and-test.yml

@@ -25,10 +25,12 @@ jobs:
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+        with:
+          fetch-depth: 0
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/check-datahub-jars.yml

@@ -30,10 +30,10 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/docker-unified.yml

@@ -359,10 +359,10 @@ jobs:
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/documentation.yml

@@ -19,10 +19,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.10"

.github/workflows/metadata-ingestion.yml

@@ -20,15 +20,14 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-
   metadata-ingestion-general:
     runs-on: ubuntu-latest
     env:
       SPARK_VERSION: 3.0.3
       DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
-        python-version: ["3.7", "3.10"]
+        python-version: ["3.7", "3.10.6"]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
@@ -56,7 +55,13 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.7", "3.10"]
-        command: ["installAirflow1", "testIntegration", "testIntegrationBatch1", "testSlowIntegration"]
+        command:
+          [
+            "installAirflow1",
+            "testIntegration",
+            "testIntegrationBatch1",
+            "testSlowIntegration",
+          ]
       fail-fast: false
     steps:
       - uses: actions/checkout@v2
@@ -65,8 +70,8 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - uses: vemonet/setup-spark@v1 # spark is required for pyspark+pydeequ data lake profiling
         with:
-          spark-version: '3.0.3'
-          hadoop-version: '3.2'
+          spark-version: "3.0.3"
+          hadoop-version: "3.2"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Run metadata-ingestion tests

.github/workflows/metadata-io.yml

@@ -29,10 +29,10 @@ jobs:
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/publish-datahub-jars.yml

@@ -53,10 +53,10 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/spark-smoke-test.yml

@@ -28,10 +28,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.gitignore

@@ -37,7 +37,7 @@ MANIFEST
 **/build
 /config
 */i18n
-/out
+out/
 
 # Mac OS
 **/.DS_Store
@@ -68,7 +68,9 @@ metadata-ingestion/generated/**
 # docs
 docs/generated/
 tmp*
-temp*
+temp/**
 
 # frontend assets
 datahub-frontend/public/**
+
+.remote*

README.md

@@ -150,7 +150,7 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 - [Data Catalogue — Knowing your data](https://medium.com/albert-franzi/data-catalogue-knowing-your-data-15f7d0724900)
 - [DataHub: A Generalized Metadata Search & Discovery Tool](https://engineering.linkedin.com/blog/2019/data-hub)
 - [Open sourcing DataHub: LinkedIn’s metadata search and discovery platform](https://engineering.linkedin.com/blog/2020/open-sourcing-datahub--linkedins-metadata-search-and-discovery-p)
-- [Emerging Architectures for Modern Data Infrastructure](https://a16z.com/2020/10/15/the-emerging-architectures-for-modern-data-infrastructure/)
+- [Emerging Architectures for Modern Data Infrastructure](https://future.com/emerging-architectures-for-modern-data-infrastructure-2020/)
 
 See the full list [here](docs/links.md).
 

SECURITY.md

@@ -0,0 +1,18 @@
+# Reporting security issues
+
+If you think you have found a security vulnerability, please send a report to [email protected]. This address can be used for all of Acryl Data’s open source and commercial products (including but not limited to DataHub and Acryl Data). We can accept only vulnerability reports at this address.
+
+It's not mandatory, but if you'd like to encrypt your message to us; please use our PGP key. The key fingerprint is:
+
+A50B10A86CC21F4B7BE102E170764C95B4FACEBF
+
+The key is available from [keyserver.ubuntu.com](https://keyserver.ubuntu.com/pks/lookup?search=A50B10A86CC21F4B7BE102E170764C95B4FACEBF&fingerprint=on&op=index).
+
+Acryl Data will send you a response indicating the next steps in handling your report. After the initial reply to your report, the security team will keep you informed of the progress towards a fix and full announcement, and may ask for additional information or guidance.
+
+**Important:** We ask you to not disclose the vulnerability before it have been fixed and announced, unless you received a response from the Acryl Data security team that you can do so.
+
+## Security announcements
+
+We maintain [Security Advisories](https://github.com/datahub-project/datahub/security/advisories) on the DataHub project GitHub repository,
+where we will post a summary, remediation, and mitigation details for any patch containing security fixes.

build.gradle

@@ -4,16 +4,17 @@ buildscript {
   ext.mavenVersion = '3.6.3'
   ext.springVersion = '5.3.20'
   ext.springBootVersion = '2.5.12'
+  ext.openTelemetryVersion = '1.18.0'
+  ext.neo4jVersion = '4.4.9'
+  ext.graphQLJavaVersion = '19.0'
+  ext.testContainersVersion = '1.17.4'
   apply from: './repositories.gradle'
   buildscript.repositories.addAll(project.repositories)
   dependencies {
     classpath 'com.linkedin.pegasus:gradle-plugins:' + pegasusVersion
     classpath 'com.github.node-gradle:gradle-node-plugin:2.2.4'
     classpath 'com.commercehub.gradle.plugin:gradle-avro-plugin:0.8.1'
     classpath 'org.springframework.boot:spring-boot-gradle-plugin:' + springBootVersion
-    classpath('com.github.jengelman.gradle.plugins:shadow:5.2.0') {
-      exclude group: 'org.apache.logging.log4j', module: 'log4j-core'
-    }
     classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0"
     classpath "com.palantir.gradle.gitversion:gradle-git-version:0.12.3"
     classpath "org.gradle.playframework:gradle-playframework:0.12"
@@ -23,6 +24,7 @@ buildscript {
 
 plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.0-rc2'
+  id 'com.github.johnrengelman.shadow' version '6.1.0'
 }
 
 project.ext.spec = [
@@ -50,6 +52,7 @@ project.ext.externalDependency = [
     'avroCompiler_1_7': 'org.apache.avro:avro-compiler:1.7.7',
     'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.10',
     'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.1',
+    'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.8',
     'cacheApi' : 'javax.cache:cache-api:1.1.0',
     'commonsCli': 'commons-cli:commons-cli:1.5.0',
     'commonsIo': 'commons-io:commons-io:2.4',
@@ -67,7 +70,8 @@ project.ext.externalDependency = [
     'elasticSearchRest': 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.9.3',
     'elasticSearchTransport': 'org.elasticsearch.client:transport:7.9.3',
     'findbugsAnnotations': 'com.google.code.findbugs:annotations:3.0.1',
-    'graphqlJava': 'com.graphql-java:graphql-java:16.1',
+    'graphqlJava': 'com.graphql-java:graphql-java:' + graphQLJavaVersion,
+    'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:' + graphQLJavaVersion,
     'gson': 'com.google.code.gson:gson:2.8.9',
     'guice': 'com.google.inject:guice:4.2.2',
     'guava': 'com.google.guava:guava:27.0.1-jre',
@@ -89,6 +93,8 @@ project.ext.externalDependency = [
     'jerseyGuava': 'org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1',
     'jettyJaas': 'org.eclipse.jetty:jetty-jaas:9.4.46.v20220331',
     'jgrapht': 'org.jgrapht:jgrapht-core:1.5.1',
+    'jna': 'net.java.dev.jna:jna:5.12.1',
+    'jsonPatch': 'com.github.java-json-tools:json-patch:1.13',
     'jsonSchemaAvro': 'com.github.fge:json-schema-avro:0.1.4',
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'jsonSmart': 'net.minidev:json-smart:2.4.6',
@@ -109,10 +115,10 @@ project.ext.externalDependency = [
     'mockServer': 'org.mock-server:mockserver-netty:5.11.2',
     'mockServerClient': 'org.mock-server:mockserver-client-java:5.11.2',
     'mysqlConnector': 'mysql:mysql-connector-java:8.0.20',
-    'neo4jHarness': 'org.neo4j.test:neo4j-harness:3.4.11',
-    'neo4jJavaDriver': 'org.neo4j.driver:neo4j-java-driver:4.0.1',
-    'opentelemetryApi': 'io.opentelemetry:opentelemetry-api:1.0.0',
-    'opentelemetryAnnotations': 'io.opentelemetry:opentelemetry-extension-annotations:1.0.0',
+    'neo4jHarness': 'org.neo4j.test:neo4j-harness:' + neo4jVersion,
+    'neo4jJavaDriver': 'org.neo4j.driver:neo4j-java-driver:' + neo4jVersion,
+    'opentelemetryApi': 'io.opentelemetry:opentelemetry-api:' + openTelemetryVersion,
+    'opentelemetryAnnotations': 'io.opentelemetry:opentelemetry-extension-annotations:' + openTelemetryVersion,
     'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15',
     'parquet': 'org.apache.parquet:parquet-avro:1.12.2',
     'picocli': 'info.picocli:picocli:4.5.0',
@@ -152,11 +158,11 @@ project.ext.externalDependency = [
     'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
     'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.1.12',
     'testng': 'org.testng:testng:7.3.0',
-    'testContainers': 'org.testcontainers:testcontainers:1.15.3',
-    'testContainersJunit': 'org.testcontainers:junit-jupiter:1.15.3',
-    'testContainersPostgresql':'org.testcontainers:postgresql:1.15.3',
-    'testContainersElasticsearch': 'org.testcontainers:elasticsearch:1.15.3',
-    'testContainersCassandra': 'org.testcontainers:cassandra:1.15.3',
+    'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion,
+    'testContainersJunit': 'org.testcontainers:junit-jupiter:' + testContainersVersion,
+    'testContainersPostgresql':'org.testcontainers:postgresql:' + testContainersVersion,
+    'testContainersElasticsearch': 'org.testcontainers:elasticsearch:' + testContainersVersion,
+    'testContainersCassandra': 'org.testcontainers:cassandra:' + testContainersVersion,
     'typesafeConfig':'com.typesafe:config:1.4.1',
     'wiremock':'com.github.tomakehurst:wiremock:2.10.0',
     'zookeeper': 'org.apache.zookeeper:zookeeper:3.4.14'
@@ -169,7 +175,7 @@ allprojects {
   apply plugin: 'checkstyle'
 }
 
-configure(subprojects.findAll {it.name != 'spark-lineage'}) {
+configure(subprojects.findAll {! it.name.startsWith('spark-lineage') }) {
 
   configurations.all {
     exclude group: "io.netty", module: "netty"
@@ -219,37 +225,14 @@ subprojects {
     }
   }
 
-  if (project.name != 'datahub-protobuf') {
-    tasks.withType(JavaCompile).configureEach {
-      javaCompiler = javaToolchains.compilerFor {
-        languageVersion = JavaLanguageVersion.of(8)
-      }
-    }
-    tasks.withType(Test).configureEach {
-      javaLauncher = javaToolchains.launcherFor {
-        languageVersion = JavaLanguageVersion.of(8)
-      }
-    }
-  } else {
-    tasks.withType(JavaExec).configureEach {
-      javaLauncher = javaToolchains.launcherFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
+  tasks.withType(JavaCompile).configureEach {
+    javaCompiler = javaToolchains.compilerFor {
+      languageVersion = JavaLanguageVersion.of(11)
     }
-    tasks.withType(Javadoc).configureEach {
-    javadocTool = javaToolchains.javadocToolFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
-    }
-    tasks.withType(JavaCompile).configureEach {
-      javaCompiler = javaToolchains.compilerFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
-    }
-    tasks.withType(Test).configureEach {
-      javaLauncher = javaToolchains.launcherFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
+  }
+  tasks.withType(Test).configureEach {
+    javaLauncher = javaToolchains.launcherFor {
+      languageVersion = JavaLanguageVersion.of(11)
     }
   }