Merge branch 'master' into ownership-differ

.github/workflows/build-and-test.yml

@@ -33,11 +33,7 @@ jobs:
         with:
           python-version: "3.7"
       - name: Gradle build (and test)
-        # there is some race condition in gradle build, which makes gradle never terminate in ~30% of the runs
-        # running build first without datahub-web-react:yarnBuild and then with it is 100% stable
-        # datahub-frontend:unzipAssets depends on datahub-web-react:yarnBuild but gradle does not know about it
         run: |
-          ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x datahub-web-react:yarnBuild -x datahub-frontend:unzipAssets -x :metadata-io:test -x :metadata-integration:java:datahub-protobuf:build
           ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test
       - uses: actions/upload-artifact@v2
         if: always()

.github/workflows/metadata-ingestion.yml

@@ -20,15 +20,14 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-
   metadata-ingestion-general:
     runs-on: ubuntu-latest
     env:
       SPARK_VERSION: 3.0.3
       DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
-        python-version: ["3.7", "3.10"]
+        python-version: ["3.7", "3.10.6"]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
@@ -56,7 +55,13 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.7", "3.10"]
-        command: ["installAirflow1", "testIntegration", "testIntegrationBatch1", "testSlowIntegration"]
+        command:
+          [
+            "installAirflow1",
+            "testIntegration",
+            "testIntegrationBatch1",
+            "testSlowIntegration",
+          ]
       fail-fast: false
     steps:
       - uses: actions/checkout@v2
@@ -65,8 +70,8 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - uses: vemonet/setup-spark@v1 # spark is required for pyspark+pydeequ data lake profiling
         with:
-          spark-version: '3.0.3'
-          hadoop-version: '3.2'
+          spark-version: "3.0.3"
+          hadoop-version: "3.2"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Run metadata-ingestion tests

.github/workflows/publish-datahub-jars.yml

@@ -1,4 +1,4 @@
-name: Publish Datahub Java Jars (Client, Spark Lineage, Protobuf)
+name: Publish Datahub Java Jars (Client, Spark Lineage, Protobuf, Ranger)
 
 on:
   push:
@@ -142,7 +142,8 @@ jobs:
           echo signingKey=$SIGNING_KEY >> gradle.properties
           ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-integration:java:datahub-protobuf:publish
           ./gradlew :metadata-integration:java:datahub-protobuf:closeAndReleaseRepository --info
-      - name: build and publish datahub-ranger-plugin
+      - name: publish datahub-ranger-plugin snapshot jar
+        if: ${{ github.event_name != 'release' }}
         env:
           RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}
           RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }}
@@ -152,6 +153,18 @@ jobs:
           NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
         run: |
           echo signingKey=$SIGNING_KEY >> gradle.properties
-           # Publish apache ranger plugin to maven
           ./gradlew :datahub-ranger-plugin:printVersion
-          ./gradlew :datahub-ranger-plugin:publishMavenJavaPublicationToMavenLocal
+          ./gradlew :datahub-ranger-plugin:publish
+      - name: release datahub-ranger-plugin jar
+        if: ${{ github.event_name == 'release' }}
+        env:
+          RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}
+          RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }}
+          SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }}
+          SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
+          NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }}
+          NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
+        run: |
+          echo signingKey=$SIGNING_KEY >> gradle.properties
+          ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :datahub-ranger-plugin:publish
+          ./gradlew :datahub-ranger-plugin:closeAndReleaseRepository --info

.gitignore

@@ -72,3 +72,5 @@ temp*
 
 # frontend assets
 datahub-frontend/public/**
+
+.remote*

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java

@@ -73,7 +73,8 @@ public class DashboardType implements SearchableEntityType<Dashboard, String>, B
         DOMAINS_ASPECT_NAME,
         DEPRECATION_ASPECT_NAME,
         DATA_PLATFORM_INSTANCE_ASPECT_NAME,
-        INPUT_FIELDS_ASPECT_NAME
+        INPUT_FIELDS_ASPECT_NAME,
+        SUB_TYPES_ASPECT_NAME
     );
     private static final Set<String> FACET_FIELDS = ImmutableSet.of("access", "tool");
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java

@@ -8,6 +8,7 @@
 import com.linkedin.common.InstitutionalMemory;
 import com.linkedin.common.Ownership;
 import com.linkedin.common.Status;
+import com.linkedin.common.SubTypes;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.dashboard.EditableDashboardProperties;
 import com.linkedin.data.DataMap;
@@ -86,6 +87,7 @@ public Dashboard apply(@Nonnull final EntityResponse entityResponse) {
             dataset.setDataPlatformInstance(DataPlatformInstanceAspectMapper.map(new DataPlatformInstance(dataMap))));
         mappingHelper.mapToResult(INPUT_FIELDS_ASPECT_NAME, (dashboard, dataMap) ->
             dashboard.setInputFields(InputFieldsMapper.map(new InputFields(dataMap), entityUrn)));
+        mappingHelper.mapToResult(SUB_TYPES_ASPECT_NAME, this::mapSubTypes);
 
         return mappingHelper.getResult();
     }
@@ -195,4 +197,13 @@ private void mapDomains(@Nonnull Dashboard dashboard, @Nonnull DataMap dataMap)
         final Domains domains = new Domains(dataMap);
         dashboard.setDomain(DomainAssociationMapper.map(domains, dashboard.getUrn()));
     }
+
+    private void mapSubTypes(@Nonnull Dashboard dashboard, DataMap dataMap) {
+        SubTypes pegasusSubTypes = new SubTypes(dataMap);
+        if (pegasusSubTypes.hasTypeNames()) {
+              com.linkedin.datahub.graphql.generated.SubTypes subTypes = new com.linkedin.datahub.graphql.generated.SubTypes();
+              subTypes.setTypeNames(pegasusSubTypes.getTypeNames().stream().collect(Collectors.toList()));
+              dashboard.setSubTypes(subTypes);
+        }
+    }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatform/mappers/DataPlatformMapper.java

@@ -27,7 +27,7 @@ public static DataPlatform map(@Nonnull final EntityResponse platform) {
     @Override
     public DataPlatform apply(@Nonnull final EntityResponse entityResponse) {
         final DataPlatform result = new DataPlatform();
-        final DataPlatformKey dataPlatformKey = (DataPlatformKey) EntityKeyUtils.convertUrnToEntityKey(entityResponse.getUrn(),
+        final DataPlatformKey dataPlatformKey = (DataPlatformKey) EntityKeyUtils.convertUrnToEntityKeyInternal(entityResponse.getUrn(),
             new DataPlatformKey().schema());
         result.setType(EntityType.DATA_PLATFORM);
         Urn urn = entityResponse.getUrn();

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/mappers/SchemaBlameMapper.java

@@ -83,7 +83,7 @@ public static GetSchemaBlameResult map(List<ChangeTransaction> changeTransaction
 
         SchemaFieldKey schemaFieldKey;
         try {
-          schemaFieldKey = (SchemaFieldKey) EntityKeyUtils.convertUrnToEntityKey(Urn.createFromString(schemaUrn),
+          schemaFieldKey = (SchemaFieldKey) EntityKeyUtils.convertUrnToEntityKeyInternal(Urn.createFromString(schemaUrn),
               new SchemaFieldKey().schema());
         } catch (Exception e) {
           log.debug(String.format("Could not generate schema urn for %s", schemaUrn));

datahub-graphql-core/src/main/resources/entity.graphql

@@ -4352,6 +4352,11 @@ type Dashboard implements EntityWithRelationships & Entity & BrowsableEntity {
     Input fields that power all the charts in the dashboard
     """
     inputFields: InputFields
+
+    """
+    Sub Types of the dashboard
+    """
+    subTypes: SubTypes
 }
 
 """

datahub-ranger-plugin/build.gradle

@@ -1,14 +1,8 @@
-import org.apache.tools.ant.filters.ReplaceTokens
-
-
 apply plugin: 'java'
 apply plugin: 'signing'
 apply plugin: 'maven-publish'
 apply plugin: 'io.codearte.nexus-staging'
-
-
-
-import org.apache.tools.ant.filters.ReplaceTokens
+apply from: '../metadata-integration/java/versioning.gradle'
 
 
 repositories {
@@ -31,53 +25,6 @@ dependencies {
     testCompile externalDependency.testng
 }
 
-def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
-def snapshotVersion = false
-if (project.hasProperty("releaseVersion")) {
-    version = releaseVersion
-    detailedVersionString = releaseVersion
-} else {
-    try {
-        // apply this plugin in a try-catch block so that we can handle cases without .git directory
-        apply plugin: "com.palantir.git-version"
-        def details = versionDetails()
-        detailedVersionString = gitVersion()
-        version = details.lastTag
-        version = version.startsWith("v")? version.substring(1): version
-        def suffix = details.isCleanTag? "": "-SNAPSHOT"
-        snapshotVersion = ! details.isCleanTag
-    }
-    catch (Exception e) {
-        e.printStackTrace()
-        // last fall back
-        version = detailedVersionString
-    }
-}
-// trim version if it is of size 4 to size 3
-def versionParts = version.tokenize(".")
-if (versionParts.size() > 3) {
-    // at-least 4 part version
-    // we check if the 4th part is a .0 in which case we want to create a release
-    if (versionParts[3] != '0') {
-        snapshotVersion = true
-    }
-    versionParts = versionParts[0..2]
-    version = versionParts[0..2].join('.')
-}
-
-if (snapshotVersion) {
-    if (versionParts[versionParts.size()-1].isInteger()) {
-        version = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT"
-    } else {
-        // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
-        version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
-    }
-}
-
-processResources {
-    filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString])
-}
-
 
 publishing {
     publications {

datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx

@@ -22,6 +22,7 @@ import { getDataForEntityType } from '../shared/containers/profile/utils';
 import { SidebarDomainSection } from '../shared/containers/profile/sidebar/Domain/SidebarDomainSection';
 import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown';
 import { LineageTab } from '../shared/tabs/Lineage/LineageTab';
+import { capitalizeFirstLetterOnly } from '../../shared/textUtil';
 import { DashboardStatsSummarySubHeader } from './profile/DashboardStatsSummarySubHeader';
 import { ChartSnippet } from '../chart/ChartSnippet';
 
@@ -153,9 +154,11 @@ export class DashboardEntity implements Entity<Dashboard> {
         // TODO: Get rid of this once we have correctly formed platform coming back.
         const name = dashboard?.properties?.name;
         const externalUrl = dashboard?.properties?.externalUrl;
+        const subTypes = dashboard?.subTypes;
         return {
             name,
             externalUrl,
+            entityTypeOverride: subTypes ? capitalizeFirstLetterOnly(subTypes.typeNames?.[0]) : '',
         };
     };
 
@@ -179,6 +182,7 @@ export class DashboardEntity implements Entity<Dashboard> {
                 statsSummary={data.statsSummary}
                 lastUpdatedMs={data.properties?.lastModified?.time}
                 createdMs={data.properties?.created?.time}
+                subtype={data.subTypes?.typeNames?.[0]}
             />
         );
     };
@@ -214,6 +218,7 @@ export class DashboardEntity implements Entity<Dashboard> {
                         inputFields={data.inputFields}
                     />
                 }
+                subtype={data.subTypes?.typeNames?.[0]}
             />
         );
     };
@@ -223,6 +228,7 @@ export class DashboardEntity implements Entity<Dashboard> {
             urn: entity.urn,
             name: entity.properties?.name || '',
             type: EntityType.Dashboard,
+            subtype: entity?.subTypes?.typeNames?.[0] || undefined,
             icon: entity?.platform?.properties?.logoUrl || '',
             platform: entity?.platform,
         };

datahub-web-react/src/app/entity/dashboard/preview/DashboardPreview.tsx

@@ -14,16 +14,17 @@ import {
 } from '../../../../types.generated';
 import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
 import { useEntityRegistry } from '../../../useEntityRegistry';
-import { capitalizeFirstLetter } from '../../../shared/textUtil';
+import { capitalizeFirstLetter, capitalizeFirstLetterOnly } from '../../../shared/textUtil';
 import { IconStyleType } from '../../Entity';
 import { DashboardStatsSummary as DashboardStatsSummaryView } from '../shared/DashboardStatsSummary';
 
 export const DashboardPreview = ({
     urn,
-    name,
+    platform,
     platformInstanceId,
+    name,
+    subtype,
     description,
-    platform,
     access,
     owners,
     tags,
@@ -45,6 +46,7 @@ export const DashboardPreview = ({
     platform: string;
     platformInstanceId?: string;
     name?: string;
+    subtype?: string | null;
     description?: string | null;
     access?: AccessLevel | null;
     owners?: Array<Owner> | null;
@@ -71,7 +73,7 @@ export const DashboardPreview = ({
             url={entityRegistry.getEntityUrl(EntityType.Dashboard, urn)}
             name={name || ''}
             description={description || ''}
-            type="Dashboard"
+            type={capitalizeFirstLetterOnly(subtype) || 'Dashboard'}
             typeIcon={entityRegistry.getIcon(EntityType.Dashboard, 14, IconStyleType.ACCENT)}
             logoUrl={logoUrl || ''}
             platformInstanceId={platformInstanceId}

datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx

@@ -28,6 +28,8 @@ import { OperationsTab } from './profile/OperationsTab';
 import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown';
 import { SidebarSiblingsSection } from '../shared/containers/profile/sidebar/SidebarSiblingsSection';
 import { DatasetStatsSummarySubHeader } from './profile/stats/stats/DatasetStatsSummarySubHeader';
+import { TagSummary } from './shared/TagSummary';
+import { TermSummary } from './shared/TermSummary';
 
 const SUBTYPES = {
     VIEW: 'view',
@@ -253,6 +255,19 @@ export class DatasetEntity implements Entity<Dataset> {
     renderSearch = (result: SearchResult) => {
         const data = result.entity as Dataset;
         const genericProperties = this.getGenericEntityProperties(data);
+
+        let snippet: React.ReactNode;
+
+        if (result.matchedFields.length > 0) {
+            if (result.matchedFields[0].value.includes('urn:li:tag')) {
+                snippet = <TagSummary urn={result.matchedFields[0].value} />;
+            } else if (result.matchedFields[0].value.includes('urn:li:glossaryTerm')) {
+                snippet = <TermSummary urn={result.matchedFields[0].value} />;
+            } else {
+                snippet = <b>{result.matchedFields[0].value}</b>;
+            }
+        }
+
         return (
             <Preview
                 urn={data.urn}
@@ -279,8 +294,7 @@ export class DatasetEntity implements Entity<Dataset> {
                     result.matchedFields.length > 0 &&
                     result.matchedFields.every((field) => FIELDS_TO_HIGHLIGHT.has(field.name)) && (
                         <Typography.Text>
-                            Matches {FIELDS_TO_HIGHLIGHT.get(result.matchedFields[0].name)}{' '}
-                            <b>{result.matchedFields[0].value}</b>
+                            Matches {FIELDS_TO_HIGHLIGHT.get(result.matchedFields[0].name)} {snippet}
                         </Typography.Text>
                     )
                 }

datahub-web-react/src/app/entity/dataset/profile/schema/utils/schemaTitleRenderer.tsx

@@ -7,6 +7,7 @@ import { ExtendedSchemaFields } from './types';
 import TypeLabel from '../../../../shared/tabs/Dataset/Schema/components/TypeLabel';
 import { ForeignKeyConstraint, SchemaMetadata } from '../../../../../../types.generated';
 import PrimaryKeyLabel from '../../../../shared/tabs/Dataset/Schema/components/PrimaryKeyLabel';
+import NullableLabel from '../../../../shared/tabs/Dataset/Schema/components/NullableLabel';
 import ForeignKeyLabel from '../../../../shared/tabs/Dataset/Schema/components/ForeignKeyLabel';
 
 const MAX_FIELD_PATH_LENGTH = 200;
@@ -61,6 +62,7 @@ export default function useSchemaTitleRenderer(
                     </FieldPathText>
                     <TypeLabel type={record.type} nativeDataType={record.nativeDataType} />
                     {(schemaMetadata?.primaryKeys?.includes(fieldPath) || record.isPartOfKey) && <PrimaryKeyLabel />}
+                    {record.nullable && <NullableLabel />}
                     {schemaMetadata?.foreignKeys
                         ?.filter(
                             (constraint) =>