Merge branch 'master' into fix-patch-registry

datahub-project · Sep 7, 2022 · c0b3641 · c0b3641
2 parents 4da2219 + 08622f2
commit c0b3641
Show file tree

Hide file tree

Showing 52 changed files with 2,886 additions and 1,606 deletions.
diff --git a/.github/pr-labeler-config.yml b/.github/pr-labeler-config.yml
@@ -0,0 +1,11 @@
+ingestion:
+- any: ['metadata-ingestion/**/*']
+
+devops:
+- any: ['docker/**/*', '.github/**/*']
+
+product:
+- any: ['datahub-frontend/**/*', 'datahub-graphql-core/**/*']
+
+docs:
+- any: ['docs/**/*']
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -128,7 +128,7 @@ jobs:
           publish: ${{ needs.setup.outputs.publish }}
           context: .
           file: ./docker/datahub-mae-consumer/Dockerfile
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
   mae_consumer_scan:
     name: "[Monitoring] Scan MAE consumer images for vulnerabilities"
     runs-on: ubuntu-latest
@@ -171,7 +171,7 @@ jobs:
           publish: ${{ needs.setup.outputs.publish }}
           context: .
           file: ./docker/datahub-mce-consumer/Dockerfile
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
   mce_consumer_scan:
     name: "[Monitoring] Scan MCE consumer images for vulnerabilities"
     runs-on: ubuntu-latest

diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml
@@ -0,0 +1,16 @@
+name: "Pull Request Labeler"
+on:
+  pull_request_target:
+    types: [opened, reopened]
+
+jobs:
+  triage:
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/labeler@v4
+      with:
+        repo-token: "${{ secrets.GITHUB_TOKEN }}"
+        configuration-path: '.github/pr-labeler-config.yml'
diff --git a/datahub-web-react/src/app/entity/chart/ChartSnippet.tsx b/datahub-web-react/src/app/entity/chart/ChartSnippet.tsx
@@ -9,12 +9,13 @@ import { getMatchPrioritizingPrimary } from '../shared/utils';
 type Props = {
     matchedFields: MatchedField[];
     inputFields: Maybe<InputFields> | undefined;
+    isMatchingDashboard?: boolean;
 };
 
 const LABEL_INDEX_NAME = 'fieldLabels';
 const TYPE_PROPERTY_KEY_NAME = 'type';
 
-export const ChartSnippet = ({ matchedFields, inputFields }: Props) => {
+export const ChartSnippet = ({ matchedFields, inputFields, isMatchingDashboard = false }: Props) => {
     const matchedField = getMatchPrioritizingPrimary(matchedFields, 'fieldLabels');
 
     if (matchedField?.name === LABEL_INDEX_NAME) {
@@ -36,15 +37,17 @@ export const ChartSnippet = ({ matchedFields, inputFields }: Props) => {
 
             return (
                 <Typography.Text>
-                    Matches {termType} <TagTermGroup uneditableGlossaryTerms={{ terms: [matchedGlossaryTerm] }} />
+                    Matches {termType} <TagTermGroup uneditableGlossaryTerms={{ terms: [matchedGlossaryTerm] }} />{' '}
+                    {isMatchingDashboard && 'on a contained Chart'}
                 </Typography.Text>
             );
         }
     }
 
     return matchedField ? (
         <Typography.Text>
-            Matches {FIELDS_TO_HIGHLIGHT.get(matchedField.name)} <b>{matchedField.value}</b>
+            Matches {FIELDS_TO_HIGHLIGHT.get(matchedField.name)} <b>{matchedField.value}</b>{' '}
+            {isMatchingDashboard && 'on a contained Chart'}
         </Typography.Text>
     ) : null;
 };
diff --git a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx
@@ -23,7 +23,6 @@ import { SidebarDomainSection } from '../shared/containers/profile/sidebar/Domai
 import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown';
 import { LineageTab } from '../shared/tabs/Lineage/LineageTab';
 import { DashboardStatsSummarySubHeader } from './profile/DashboardStatsSummarySubHeader';
-import { InputFieldsTab } from '../shared/tabs/Entity/InputFieldsTab';
 import { ChartSnippet } from '../chart/ChartSnippet';
 
 /**
@@ -87,16 +86,6 @@ export class DashboardEntity implements Entity<Dashboard> {
                     name: 'Documentation',
                     component: DocumentationTab,
                 },
-                {
-                    name: 'Fields',
-                    component: InputFieldsTab,
-                    display: {
-                        visible: (_, dashboard: GetDashboardQuery) =>
-                            (dashboard?.dashboard?.inputFields?.fields?.length || 0) > 0,
-                        enabled: (_, dashboard: GetDashboardQuery) =>
-                            (dashboard?.dashboard?.inputFields?.fields?.length || 0) > 0,
-                    },
-                },
                 {
                     name: 'Properties',
                     component: PropertiesTab,
@@ -215,7 +204,13 @@ export class DashboardEntity implements Entity<Dashboard> {
                 statsSummary={data.statsSummary}
                 lastUpdatedMs={data.properties?.lastModified?.time}
                 createdMs={data.properties?.created?.time}
-                snippet={<ChartSnippet matchedFields={result.matchedFields} inputFields={data.inputFields} />}
+                snippet={
+                    <ChartSnippet
+                        isMatchingDashboard
+                        matchedFields={result.matchedFields}
+                        inputFields={data.inputFields}
+                    />
+                }
             />
         );
     };

diff --git a/docker/datahub-mae-consumer/Dockerfile b/docker/datahub-mae-consumer/Dockerfile
@@ -1,15 +1,30 @@
 # Defining environment
 ARG APP_ENV=prod
 
-FROM adoptopenjdk/openjdk8:alpine-jre as base
+FROM alpine:3.14 AS base
+
 ENV DOCKERIZE_VERSION v0.6.1
-RUN apk --no-cache add curl tar wget bash coreutils \
+
+# Upgrade Alpine and base packages
+RUN apk --no-cache --update-cache --available upgrade \
+    && if [ $(arch) = "aarch64" ]; then \
+       DOCKERIZE_ARCH='aarch64';\
+    elif [ $(arch) = "x86_64" ]; then \
+      DOCKERIZE_ARCH='amd64'; \
+    else \
+      echo >&2 "Unsupported architecture $(arch)" ; exit 1; \
+    fi \
+    && apk --no-cache add tar curl bash openjdk8-jre \
     && wget --no-verbose https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.4.1/opentelemetry-javaagent-all.jar \
     && wget --no-verbose https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar -O jmx_prometheus_javaagent.jar \
-    && curl -sSL https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv
+    && curl -sSL https://github.com/treff7es/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-${DOCKERIZE_ARCH}-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv
+
+FROM --platform=$BUILDPLATFORM alpine:3.14.2 AS prod-build
+
+# Upgrade Alpine and base packages
+RUN apk --no-cache --update-cache --available upgrade \
+    && apk --no-cache add openjdk8 perl
 
-FROM adoptopenjdk/openjdk8:alpine-slim as prod-build
-RUN apk --no-cache add openjdk8-jre perl
 COPY . datahub-src
 RUN cd datahub-src && ./gradlew :metadata-jobs:mae-consumer-job:build -x test
 RUN cd datahub-src && cp metadata-jobs/mae-consumer-job/build/libs/mae-consumer-job.jar ../mae-consumer-job.jar

diff --git a/docker/datahub-mce-consumer/Dockerfile b/docker/datahub-mce-consumer/Dockerfile
@@ -1,15 +1,31 @@
 # Defining environment
 ARG APP_ENV=prod
 
-FROM adoptopenjdk/openjdk8:alpine-jre as base
+FROM alpine:3.14 AS base
+
 ENV DOCKERIZE_VERSION v0.6.1
-RUN apk --no-cache add curl tar wget openjdk8-jre bash \
+
+# Upgrade Alpine and base packages
+RUN apk --no-cache --update-cache --available upgrade \
+    && if [ $(arch) = "aarch64" ]; then \
+       DOCKERIZE_ARCH='aarch64';\
+    elif [ $(arch) = "x86_64" ]; then \
+      DOCKERIZE_ARCH='amd64'; \
+    else \
+      echo >&2 "Unsupported architecture $(arch)" ; exit 1; \
+    fi \
+    && apk --no-cache add tar curl bash openjdk8-jre \
     && wget --no-verbose https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.4.1/opentelemetry-javaagent-all.jar \
     && wget --no-verbose https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar -O jmx_prometheus_javaagent.jar \
     && cp /usr/lib/jvm/java-1.8-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks \
-    && curl -sSL https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv
+    && curl -sSL https://github.com/treff7es/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-${DOCKERIZE_ARCH}-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv
+
+FROM --platform=$BUILDPLATFORM alpine:3.14.2 AS prod-build
+
+# Upgrade Alpine and base packages
+RUN apk --no-cache --update-cache --available upgrade \
+    && apk --no-cache add openjdk8 perl
 
-FROM openjdk:8 as prod-build
 COPY . datahub-src
 RUN cd datahub-src && ./gradlew :metadata-jobs:mce-consumer-job:build
 RUN cd datahub-src && cp metadata-jobs/mce-consumer-job/build/libs/mce-consumer-job.jar ../mce-consumer-job.jar

diff --git a/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml
@@ -0,0 +1,41 @@
+services:
+  datahub-gms:
+    environment:
+    - MAE_CONSUMER_ENABLED=false
+    - MCE_CONSUMER_ENABLED=false
+  datahub-mae-consumer:
+    container_name: datahub-mae-consumer
+    depends_on:
+    - kafka-setup
+    - elasticsearch-setup
+    environment:
+    - DATAHUB_GMS_HOST=datahub-gms
+    - DATAHUB_GMS_PORT=8080
+    - MAE_CONSUMER_ENABLED=true
+    - PE_CONSUMER_ENABLED=true
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+    - ELASTICSEARCH_HOST=elasticsearch
+    - ELASTICSEARCH_PORT=9200
+    - GRAPH_SERVICE_IMPL=elasticsearch
+    - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml
+    hostname: datahub-mae-consumer
+    image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-head}
+    ports:
+    - 9091:9091
+  datahub-mce-consumer:
+    container_name: datahub-mce-consumer
+    depends_on:
+    - kafka-setup
+    - datahub-gms
+    environment:
+    - MCE_CONSUMER_ENABLED=true
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+    - DATAHUB_GMS_HOST=datahub-gms
+    - DATAHUB_GMS_PORT=8080
+    hostname: datahub-mce-consumer
+    image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-head}
+    ports:
+    - 9090:9090
+version: '2.3'
diff --git a/docker/quickstart/docker-compose.consumers.quickstart.yml b/docker/quickstart/docker-compose.consumers.quickstart.yml
@@ -0,0 +1,46 @@
+services:
+  datahub-gms:
+    environment:
+    - MAE_CONSUMER_ENABLED=false
+    - MCE_CONSUMER_ENABLED=false
+  datahub-mae-consumer:
+    container_name: datahub-mae-consumer
+    depends_on:
+    - kafka-setup
+    - elasticsearch-setup
+    - neo4j
+    environment:
+    - DATAHUB_GMS_HOST=datahub-gms
+    - DATAHUB_GMS_PORT=8080
+    - MAE_CONSUMER_ENABLED=true
+    - PE_CONSUMER_ENABLED=true
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+    - ELASTICSEARCH_HOST=elasticsearch
+    - ELASTICSEARCH_PORT=9200
+    - NEO4J_HOST=http://neo4j:7474
+    - NEO4J_URI=bolt://neo4j
+    - NEO4J_USERNAME=neo4j
+    - NEO4J_PASSWORD=datahub
+    - GRAPH_SERVICE_IMPL=neo4j
+    - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml
+    hostname: datahub-mae-consumer
+    image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-head}
+    ports:
+    - 9091:9091
+  datahub-mce-consumer:
+    container_name: datahub-mce-consumer
+    depends_on:
+    - kafka-setup
+    - datahub-gms
+    environment:
+    - MCE_CONSUMER_ENABLED=true
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+    - DATAHUB_GMS_HOST=datahub-gms
+    - DATAHUB_GMS_PORT=8080
+    hostname: datahub-mce-consumer
+    image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-head}
+    ports:
+    - 9090:9090
+version: '2.3'
diff --git a/docker/quickstart/generate_and_compare.sh b/docker/quickstart/generate_and_compare.sh
@@ -15,6 +15,8 @@ pip install -r requirements.txt
 python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml temp.quickstart.yml
 python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml temp-without-neo4j.quickstart.yml
 python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml temp.monitoring.quickstart.yml
+python generate_docker_quickstart.py ../docker-compose.consumers.yml temp.consumers.quickstart.yml
+python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml temp.consumers-without-neo4j.quickstart.yml
 
 for flavour in "${FLAVOURS[@]}"
 do

diff --git a/docker/quickstart/generate_docker_quickstart.sh b/docker/quickstart/generate_docker_quickstart.sh
@@ -12,3 +12,5 @@ pip install -r requirements.txt
 python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml docker-compose.quickstart.yml
 python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml docker-compose-without-neo4j.quickstart.yml
 python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml docker-compose.monitoring.quickstart.yml
+python generate_docker_quickstart.py ../docker-compose.consumers.yml docker-compose.consumers.quickstart.yml
+python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml docker-compose.consumers-without-neo4j.quickstart.yml
diff --git a/docs-website/genJsonSchema/gen_json_schema.py b/docs-website/genJsonSchema/gen_json_schema.py
@@ -159,7 +159,7 @@ def get_base() -> Any:
                 "type": "array",
                 "items": {
                     "type": "object",
-                    "description": "Transformer configs see at https://datahubproject.io/docs/metadata-ingestion/transformers",
+                    "description": "Transformer configs see at https://datahubproject.io/docs/metadata-ingestion/docs/transformer",
                     "properties": {
                         "type": {"type": "string", "description": "Transformer type"},
                         "config": {

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
@@ -137,7 +137,12 @@ module.exports = {
       {
         Sinks: list_ids_in_directory("metadata-ingestion/sink_docs"),
       },
-      "metadata-ingestion/transformers",
+      {
+        Transformers: [
+          "metadata-ingestion/docs/transformer/intro",
+          "metadata-ingestion/docs/transformer/dataset_transformer",
+        ],
+      },
       {
         "Advanced Guides": [
           {

diff --git a/metadata-ingestion/README.md b/metadata-ingestion/README.md
@@ -183,7 +183,7 @@ transformers: # an array of transformers applied sequentially
 # default sink, no config needed
 ```
 
-Check out the [transformers guide](./transformers.md) to learn more about how you can create really flexible pipelines for processing metadata using Transformers!
+Check out the [transformers guide](./docs/transformer/intro.md) to learn more about how you can create really flexible pipelines for processing metadata using Transformers!
 
 ## Using as a library (SDK)
 
@@ -195,5 +195,5 @@ In some cases, you might want to configure and run a pipeline entirely from with
 
 ## Developing
 
-See the guides on [developing](./developing.md), [adding a source](./adding-source.md) and [using transformers](./transformers.md).
+See the guides on [developing](./developing.md), [adding a source](./adding-source.md) and [using transformers](./docs/transformer/intro.md).
 
diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle
@@ -8,7 +8,7 @@ ext {
 }
 
 task checkPythonVersion(type: Exec) {
-  commandLine python_executable, '-c', 'import sys; assert sys.version_info >= (3, 7)'
+  commandLine python_executable, '-c', 'import sys; assert sys.version_info >= (3, 6)'
 }
 
 task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {