refactor(java11) - convert most modules to java 11

.github/workflows/build-and-test.yml

@@ -25,10 +25,10 @@ jobs:
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/check-datahub-jars.yml

@@ -30,10 +30,10 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/docker-unified.yml

@@ -359,10 +359,10 @@ jobs:
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/documentation.yml

@@ -19,10 +19,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.10"

.github/workflows/metadata-io.yml

@@ -29,10 +29,10 @@ jobs:
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/publish-datahub-jars.yml

@@ -53,10 +53,10 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.github/workflows/spark-smoke-test.yml

@@ -28,10 +28,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"

.gitignore

@@ -37,7 +37,7 @@ MANIFEST
 **/build
 /config
 */i18n
-/out
+out/
 
 # Mac OS
 **/.DS_Store

build.gradle

@@ -11,9 +11,6 @@ buildscript {
     classpath 'com.github.node-gradle:gradle-node-plugin:2.2.4'
     classpath 'com.commercehub.gradle.plugin:gradle-avro-plugin:0.8.1'
     classpath 'org.springframework.boot:spring-boot-gradle-plugin:' + springBootVersion
-    classpath('com.github.jengelman.gradle.plugins:shadow:5.2.0') {
-      exclude group: 'org.apache.logging.log4j', module: 'log4j-core'
-    }
     classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0"
     classpath "com.palantir.gradle.gitversion:gradle-git-version:0.12.3"
     classpath "org.gradle.playframework:gradle-playframework:0.12"
@@ -23,6 +20,7 @@ buildscript {
 
 plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.0-rc2'
+  id 'com.github.johnrengelman.shadow' version '6.1.0'
 }
 
 project.ext.spec = [
@@ -171,7 +169,7 @@ allprojects {
   apply plugin: 'checkstyle'
 }
 
-configure(subprojects.findAll {it.name != 'spark-lineage'}) {
+configure(subprojects.findAll {! it.name.startsWith('spark-lineage') }) {
 
   configurations.all {
     exclude group: "io.netty", module: "netty"
@@ -221,37 +219,14 @@ subprojects {
     }
   }
 
-  if (project.name != 'datahub-protobuf') {
-    tasks.withType(JavaCompile).configureEach {
-      javaCompiler = javaToolchains.compilerFor {
-        languageVersion = JavaLanguageVersion.of(8)
-      }
-    }
-    tasks.withType(Test).configureEach {
-      javaLauncher = javaToolchains.launcherFor {
-        languageVersion = JavaLanguageVersion.of(8)
-      }
-    }
-  } else {
-    tasks.withType(JavaExec).configureEach {
-      javaLauncher = javaToolchains.launcherFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
+  tasks.withType(JavaCompile).configureEach {
+    javaCompiler = javaToolchains.compilerFor {
+      languageVersion = JavaLanguageVersion.of(11)
     }
-    tasks.withType(Javadoc).configureEach {
-    javadocTool = javaToolchains.javadocToolFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
-    }
-    tasks.withType(JavaCompile).configureEach {
-      javaCompiler = javaToolchains.compilerFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
-    }
-    tasks.withType(Test).configureEach {
-      javaLauncher = javaToolchains.launcherFor {
-        languageVersion = JavaLanguageVersion.of(11)
-      }
+  }
+  tasks.withType(Test).configureEach {
+    javaLauncher = javaToolchains.launcherFor {
+      languageVersion = JavaLanguageVersion.of(11)
     }
   }
 

docker/broker/env/docker.env

@@ -5,3 +5,4 @@ KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9
 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
 KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
+KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false

docker/datahub-frontend/Dockerfile

@@ -14,7 +14,7 @@ FROM --platform=$BUILDPLATFORM node:16.13.0-alpine3.14 AS prod-build
 
 # Upgrade Alpine and base packages
 RUN apk --no-cache --update-cache --available upgrade \
-    && apk --no-cache add perl openjdk8
+    && apk --no-cache add perl openjdk8 openjdk11
 
 ARG USE_SYSTEM_NODE="true"
 ENV CI=true

docker/datahub-gms/Dockerfile

@@ -28,7 +28,7 @@ FROM --platform=$BUILDPLATFORM alpine:3.14 AS prod-build
 
 # Upgrade Alpine and base packages
 RUN apk --no-cache --update-cache --available upgrade \
-    && apk --no-cache add openjdk8 perl
+    && apk --no-cache add openjdk8 openjdk11 perl
 
 COPY . /datahub-src
 RUN cd /datahub-src && ./gradlew :metadata-service:war:build -x test

docker/datahub-ingestion/Dockerfile

@@ -3,7 +3,7 @@ ARG APP_ENV=prod
 
 FROM acryldata/datahub-ingestion-base as base
 
-FROM openjdk:8 as prod-build
+FROM openjdk:11 as prod-build
 COPY . /datahub-src
 RUN cd /datahub-src && ./gradlew :metadata-events:mxe-schemas:build
 

docker/datahub-mae-consumer/Dockerfile

@@ -25,7 +25,7 @@ FROM --platform=$BUILDPLATFORM alpine:3.14.2 AS prod-build
 
 # Upgrade Alpine and base packages
 RUN apk --no-cache --update-cache --available upgrade \
-    && apk --no-cache add openjdk8 perl
+    && apk --no-cache add openjdk8 openjdk11 perl
 
 COPY . datahub-src
 RUN cd datahub-src && ./gradlew :metadata-jobs:mae-consumer-job:build -x test

docker/datahub-mce-consumer/Dockerfile

@@ -25,7 +25,7 @@ FROM --platform=$BUILDPLATFORM alpine:3.14.2 AS prod-build
 
 # Upgrade Alpine and base packages
 RUN apk --no-cache --update-cache --available upgrade \
-    && apk --no-cache add openjdk8 perl
+    && apk --no-cache add openjdk8 openjdk11 perl
 
 COPY . datahub-src
 RUN cd datahub-src && ./gradlew :metadata-jobs:mce-consumer-job:build

docker/datahub-upgrade/Dockerfile

@@ -26,7 +26,7 @@ FROM --platform=$BUILDPLATFORM alpine:3.14 AS prod-build
 
 # Upgrade Alpine and base packages
 RUN apk --no-cache --update-cache --available upgrade \
-    && apk --no-cache add openjdk8 perl
+    && apk --no-cache add openjdk8 openjdk11 perl
 
 COPY . datahub-src
 RUN cd datahub-src && ./gradlew :datahub-upgrade:build

docker/docker-compose-with-cassandra.yml

@@ -14,7 +14,7 @@ services:
     ports:
       - "2181:2181"
     volumes:
-      - zkdata:/var/opt/zookeeper
+      - zkdata:/var/lib/zookeeper
 
   broker:
     image: confluentinc/cp-kafka:5.4.0

docker/docker-compose-without-neo4j.yml

@@ -14,7 +14,7 @@ services:
     ports:
       - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     volumes:
-      - zkdata:/var/opt/zookeeper
+      - zkdata:/var/lib/zookeeper
 
   broker:
     image: confluentinc/cp-kafka:5.4.0

docker/docker-compose.yml

@@ -14,7 +14,7 @@ services:
     ports:
       - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     volumes:
-      - zkdata:/var/opt/zookeeper
+      - zkdata:/var/lib/zookeeper
 
   broker:
     image: confluentinc/cp-kafka:5.4.0

docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml

@@ -14,6 +14,7 @@ services:
     - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
     - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
     - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
+    - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
     hostname: broker
     image: confluentinc/cp-kafka:7.2.0
     ports:
@@ -176,7 +177,7 @@ services:
     ports:
     - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     volumes:
-    - zkdata:/var/opt/zookeeper
+    - zkdata:/var/lib/zookeeper
 version: '2.3'
 volumes:
   esdata: null

docker/quickstart/docker-compose-without-neo4j.quickstart.yml

@@ -14,6 +14,7 @@ services:
     - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
     - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
     - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
+    - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
     hostname: broker
     image: confluentinc/cp-kafka:5.4.0
     ports:
@@ -178,7 +179,7 @@ services:
     ports:
     - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     volumes:
-    - zkdata:/var/opt/zookeeper
+    - zkdata:/var/lib/zookeeper
 version: '2.3'
 volumes:
   esdata: null

docker/quickstart/docker-compose.quickstart.yml

@@ -14,6 +14,7 @@ services:
     - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
     - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
     - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
+    - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
     hostname: broker
     image: confluentinc/cp-kafka:5.4.0
     ports:
@@ -198,7 +199,7 @@ services:
     ports:
     - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     volumes:
-    - zkdata:/var/opt/zookeeper
+    - zkdata:/var/lib/zookeeper
 version: '2.3'
 volumes:
   broker: null

docs/developers.md

@@ -5,14 +5,14 @@ title: "Local Development"
 # DataHub Developer's Guide
 
 ## Pre-requirements
- - [Java 1.8 SDK](https://adoptopenjdk.net/?variant=openjdk8&jvmVariant=hotspot)
+ - [Java 11 SDK](https://openjdk.org/projects/jdk/11/)
  - [Docker](https://www.docker.com/)
  - [Docker Compose](https://docs.docker.com/compose/)
  - Docker engine with at least 8GB of memory to run tests.
 
  :::note
 
- Do not try to use a JDK newer than JDK 8. The build process does not work with newer JDKs currently.
+ Do not try to use a JDK newer than JDK 11. The build process does not work with newer JDKs currently.
 
  :::
 
@@ -101,7 +101,7 @@ You're probably using a Java version that's too new for gradle. Run the followin
 ```
 java --version
 ```
-While it may be possible to build and run DataHub using newer versions of Java, we currently only support [Java 1.8](https://www.oracle.com/java/technologies/javase/javase-jdk8-downloads.html) (aka Java 8). Plan for Java 11 migration is being discussed in [this issue](https://github.com/datahub-project/datahub/issues/1699).
+While it may be possible to build and run DataHub using newer versions of Java, we currently only support [Java 11](https://openjdk.org/projects/jdk/11/) (aka Java 11).
 
 ### Getting `cannot find symbol` error for `javax.annotation.Generated`
 

docs/how/updating-datahub.md

@@ -5,6 +5,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 ## Next
 
 ### Breaking Changes
+- Java version 11 or greater is required.
 
 ### Potential Downtime
 

li-utils/build.gradle

@@ -1,6 +1,17 @@
 apply plugin: 'java'
 apply plugin: 'pegasus'
 
+tasks.withType(JavaCompile).configureEach {
+    javaCompiler = javaToolchains.compilerFor {
+        languageVersion = JavaLanguageVersion.of(8)
+    }
+}
+tasks.withType(Test).configureEach {
+    javaLauncher = javaToolchains.launcherFor {
+        languageVersion = JavaLanguageVersion.of(8)
+    }
+}
+
 dependencies {
     compile spec.product.pegasus.data
     compile externalDependency.commonsLang