Merge branch 'master' into disable_preflight_script

treff7es · Nov 29, 2024 · 49280fc · 49280fc
2 parents de4133a + a46de1e
commit 49280fc
Show file tree

Hide file tree

Showing 173 changed files with 12,839 additions and 3,079 deletions.
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -83,6 +83,7 @@ jobs:
       - uses: gradle/actions/setup-gradle@v3
       - name: Gradle build (and test) for NOT metadata ingestion
         if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
+        # datahub-schematron:cli excluded due to dependency on metadata-ingestion
         run: |
           ./gradlew build \
             -x :metadata-ingestion:build \
@@ -100,6 +101,7 @@ jobs:
             -x :metadata-ingestion-modules:gx-plugin:check \
             -x :datahub-frontend:build \
             -x :datahub-web-react:build \
+            -x :metadata-integration:java:datahub-schematron:cli:test \
             --parallel
       - name: Gradle build (and test) for frontend
         if: ${{  matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }}

diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
@@ -40,4 +40,5 @@ jobs:
       - name: check ${{ matrix.command }} jar
         run: |
           ./gradlew :metadata-integration:java:${{ matrix.command }}:build --info
+          ./gradlew :metadata-integration:java:${{ matrix.command }}:checkShadowJar
           ./gradlew :metadata-integration:java:${{ matrix.command }}:javadoc
diff --git a/build.gradle b/build.gradle
@@ -48,6 +48,7 @@ buildscript {
   // see also datahub-frontend/play.gradle
   ext.playVersion = '2.8.22'
   ext.playScalaVersion = '2.13'
+  ext.akkaVersion = '2.6.21' // 2.7.0+ has incompatible license
   ext.log4jVersion = '2.23.1'
   ext.slf4jVersion = '1.7.36'
   ext.logbackClassic = '1.4.14'
@@ -105,7 +106,14 @@ project.ext.spec = [
 ]
 
 project.ext.externalDependency = [
-    'akkaHttp': "com.typesafe.akka:akka-http-core_$playScalaVersion:10.2.10",
+    'akkaHttp': "com.typesafe.akka:akka-http-core_$playScalaVersion:10.2.10", // max version due to licensing
+    'akkaActor': "com.typesafe.akka:akka-actor_$playScalaVersion:$akkaVersion",
+    'akkaStream': "com.typesafe.akka:akka-stream_$playScalaVersion:$akkaVersion",
+    'akkaActorTyped': "com.typesafe.akka:akka-actor-typed_$playScalaVersion:$akkaVersion",
+    'akkaSlf4j': "com.typesafe.akka:akka-slf4j_$playScalaVersion:$akkaVersion",
+    'akkaJackson': "com.typesafe.akka:akka-serialization-jackson_$playScalaVersion:$akkaVersion",
+    'akkaParsing': "com.typesafe.akka:akka-parsing_$playScalaVersion:$akkaVersion",
+    'akkaProtobuf': "com.typesafe.akka:akka-protobuf-v3_$playScalaVersion:$akkaVersion",
     'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3',
     'antlr4': 'org.antlr:antlr4:4.9.3',
     'assertJ': 'org.assertj:assertj-core:3.11.1',
@@ -350,6 +358,7 @@ allprojects {
       }
     }
   }
+
 }
 
 configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {

diff --git a/datahub-frontend/play.gradle b/datahub-frontend/play.gradle
@@ -55,6 +55,13 @@ dependencies {
   implementation externalDependency.antlr4Runtime
   implementation externalDependency.antlr4
   implementation externalDependency.akkaHttp
+  implementation externalDependency.akkaActor
+  implementation externalDependency.akkaStream
+  implementation externalDependency.akkaActorTyped
+  implementation externalDependency.akkaSlf4j
+  implementation externalDependency.akkaJackson
+  implementation externalDependency.akkaParsing
+  implementation externalDependency.akkaProtobuf
 
   implementation externalDependency.jerseyCore
   implementation externalDependency.jerseyGuava

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
@@ -63,6 +63,7 @@
 import com.linkedin.datahub.graphql.generated.Domain;
 import com.linkedin.datahub.graphql.generated.ERModelRelationship;
 import com.linkedin.datahub.graphql.generated.ERModelRelationshipProperties;
+import com.linkedin.datahub.graphql.generated.Entity;
 import com.linkedin.datahub.graphql.generated.EntityPath;
 import com.linkedin.datahub.graphql.generated.EntityRelationship;
 import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy;
@@ -312,6 +313,7 @@
 import com.linkedin.datahub.graphql.resolvers.type.HyperParameterValueTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.type.PlatformSchemaUnionTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.type.PropertyValueResolver;
+import com.linkedin.datahub.graphql.resolvers.type.ResolvedActorResolver;
 import com.linkedin.datahub.graphql.resolvers.type.ResultsTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.type.TimeSeriesAspectInterfaceTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.user.CreateNativeUserResetTokenResolver;
@@ -1730,12 +1732,22 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) {
         .type(
             "InstitutionalMemoryMetadata",
             typeWiring ->
-                typeWiring.dataFetcher(
-                    "author",
-                    new LoadableTypeResolver<>(
-                        corpUserType,
-                        (env) ->
-                            ((InstitutionalMemoryMetadata) env.getSource()).getAuthor().getUrn())))
+                typeWiring
+                    .dataFetcher(
+                        "author",
+                        new LoadableTypeResolver<>(
+                            corpUserType,
+                            (env) ->
+                                ((InstitutionalMemoryMetadata) env.getSource())
+                                    .getAuthor()
+                                    .getUrn()))
+                    .dataFetcher(
+                        "actor",
+                        new EntityTypeResolver(
+                            this.entityTypes,
+                            (env) ->
+                                (Entity)
+                                    ((InstitutionalMemoryMetadata) env.getSource()).getActor())))
         .type(
             "DatasetStatsSummary",
             typeWiring ->
@@ -2242,6 +2254,7 @@ private void configureTypeResolvers(final RuntimeWiring.Builder builder) {
             "HyperParameterValueType",
             typeWiring -> typeWiring.typeResolver(new HyperParameterValueTypeResolver()))
         .type("PropertyValue", typeWiring -> typeWiring.typeResolver(new PropertyValueResolver()))
+        .type("ResolvedActor", typeWiring -> typeWiring.typeResolver(new ResolvedActorResolver()))
         .type("Aspect", typeWiring -> typeWiring.typeResolver(new AspectInterfaceTypeResolver()))
         .type(
             "TimeSeriesAspect",

diff --git a/...core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java b/...core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java
@@ -84,8 +84,21 @@ private TimeSeriesChart getActiveUsersTimeSeriesChart(
       final DateTime end,
       final String title,
       final DateInterval interval) {
-    final DateRange dateRange =
-        new DateRange(String.valueOf(beginning.getMillis()), String.valueOf(end.getMillis()));
+
+    final DateRange dateRange;
+
+    // adjust month to show 1st of month rather than last day of previous month
+    if (interval == DateInterval.MONTH) {
+      dateRange =
+          new DateRange(
+              String.valueOf(beginning.plusDays(1).getMillis()), // Shift start by 1 day
+              String.valueOf(end.plusDays(1).getMillis()) // Shift end by 1 day
+              );
+    } else {
+      // week display starting Sundays
+      dateRange =
+          new DateRange(String.valueOf(beginning.getMillis()), String.valueOf(end.getMillis()));
+    }
 
     final List<NamedLine> timeSeriesLines =
         _analyticsService.getTimeseriesChart(
@@ -96,6 +109,7 @@ private TimeSeriesChart getActiveUsersTimeSeriesChart(
             ImmutableMap.of(),
             Collections.emptyMap(),
             Optional.of("browserId"));
+
     return TimeSeriesChart.builder()
         .setTitle(title)
         .setDateRange(dateRange)

diff --git a/...core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/ResolvedActorResolver.java b/...core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/ResolvedActorResolver.java
@@ -0,0 +1,25 @@
+package com.linkedin.datahub.graphql.resolvers.type;
+
+import com.linkedin.datahub.graphql.generated.CorpGroup;
+import com.linkedin.datahub.graphql.generated.CorpUser;
+import graphql.TypeResolutionEnvironment;
+import graphql.schema.GraphQLObjectType;
+import graphql.schema.TypeResolver;
+
+public class ResolvedActorResolver implements TypeResolver {
+
+  public static final String CORP_USER = "CorpUser";
+  public static final String CORP_GROUP = "CorpGroup";
+
+  @Override
+  public GraphQLObjectType getType(TypeResolutionEnvironment env) {
+    if (env.getObject() instanceof CorpUser) {
+      return env.getSchema().getObjectType(CORP_USER);
+    } else if (env.getObject() instanceof CorpGroup) {
+      return env.getSchema().getObjectType(CORP_GROUP);
+    } else {
+      throw new RuntimeException(
+          "Unrecognized object type provided to type resolver, Type:" + env.getObject().toString());
+    }
+  }
+}
diff --git a/.../com/linkedin/datahub/graphql/types/common/mappers/InstitutionalMemoryMetadataMapper.java b/.../com/linkedin/datahub/graphql/types/common/mappers/InstitutionalMemoryMetadataMapper.java
@@ -28,6 +28,7 @@ public InstitutionalMemoryMetadata apply(
     result.setDescription(input.getDescription()); // deprecated field
     result.setLabel(input.getDescription());
     result.setAuthor(getAuthor(input.getCreateStamp().getActor().toString()));
+    result.setActor(ResolvedActorMapper.map(input.getCreateStamp().getActor()));
     result.setCreated(AuditStampMapper.map(context, input.getCreateStamp()));
     result.setAssociatedUrn(entityUrn.toString());
     return result;

diff --git a/.../src/main/java/com/linkedin/datahub/graphql/types/common/mappers/ResolvedActorMapper.java b/.../src/main/java/com/linkedin/datahub/graphql/types/common/mappers/ResolvedActorMapper.java
@@ -0,0 +1,31 @@
+package com.linkedin.datahub.graphql.types.common.mappers;
+
+import com.linkedin.common.urn.Urn;
+import com.linkedin.datahub.graphql.generated.CorpGroup;
+import com.linkedin.datahub.graphql.generated.CorpUser;
+import com.linkedin.datahub.graphql.generated.EntityType;
+import com.linkedin.datahub.graphql.generated.ResolvedActor;
+import com.linkedin.metadata.Constants;
+import javax.annotation.Nonnull;
+
+public class ResolvedActorMapper {
+
+  public static final ResolvedActorMapper INSTANCE = new ResolvedActorMapper();
+
+  public static ResolvedActor map(@Nonnull final Urn actorUrn) {
+    return INSTANCE.apply(actorUrn);
+  }
+
+  public ResolvedActor apply(@Nonnull final Urn actorUrn) {
+    if (actorUrn.getEntityType().equals(Constants.CORP_GROUP_ENTITY_NAME)) {
+      CorpGroup partialGroup = new CorpGroup();
+      partialGroup.setUrn(actorUrn.toString());
+      partialGroup.setType(EntityType.CORP_GROUP);
+      return partialGroup;
+    }
+    CorpUser partialUser = new CorpUser();
+    partialUser.setUrn(actorUrn.toString());
+    partialUser.setType(EntityType.CORP_USER);
+    return (ResolvedActor) partialUser;
+  }
+}
diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql
@@ -3005,8 +3005,14 @@ type InstitutionalMemoryMetadata {
 
     """
     The author of this metadata
+    Deprecated! Use actor instead for users or groups.
     """
-    author: CorpUser!
+    author: CorpUser! @deprecated(reason: "Use `actor`")
+
+    """
+    The author of this metadata
+    """
+    actor: ResolvedActor!
 
     """
     An AuditStamp corresponding to the creation of this resource
@@ -3834,6 +3840,8 @@ enum CorpUserStatus {
   ACTIVE
 }
 
+union ResolvedActor = CorpUser | CorpGroup
+
 """
 A DataHub User entity, which represents a Person on the Metadata Entity Graph
 """

diff --git a/...va/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java b/...va/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java
@@ -386,7 +386,11 @@ private static EntityClient initMockEntityClient(
     Mockito.when(
             client.searchAcrossEntities(
                 any(),
-                Mockito.eq(entityTypes),
+                Mockito.argThat(
+                    argument ->
+                        argument != null
+                            && argument.containsAll(entityTypes)
+                            && entityTypes.containsAll(argument)),
                 Mockito.eq(query),
                 Mockito.eq(filter),
                 Mockito.eq(start),
@@ -409,7 +413,11 @@ private static void verifyMockEntityClient(
     Mockito.verify(mockClient, Mockito.times(1))
         .searchAcrossEntities(
             any(),
-            Mockito.eq(entityTypes),
+            Mockito.argThat(
+                argument ->
+                    argument != null
+                        && argument.containsAll(entityTypes)
+                        && entityTypes.containsAll(argument)),
             Mockito.eq(query),
             Mockito.eq(filter),
             Mockito.eq(start),

diff --git a/.../java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java b/.../java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java
@@ -462,7 +462,11 @@ private static EntityClient initMockEntityClient(
     Mockito.when(
             client.searchAcrossEntities(
                 any(),
-                Mockito.eq(entityTypes),
+                Mockito.argThat(
+                    argument ->
+                        argument != null
+                            && argument.containsAll(entityTypes)
+                            && entityTypes.containsAll(argument)),
                 Mockito.eq(query),
                 Mockito.eq(filter),
                 Mockito.eq(start),
@@ -483,7 +487,11 @@ private static void verifyMockEntityClient(
     Mockito.verify(mockClient, Mockito.times(1))
         .searchAcrossEntities(
             any(),
-            Mockito.eq(entityTypes),
+            Mockito.argThat(
+                argument ->
+                    argument != null
+                        && argument.containsAll(entityTypes)
+                        && entityTypes.containsAll(argument)),
             Mockito.eq(query),
             Mockito.eq(filter),
             Mockito.eq(start),

diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java
@@ -47,4 +47,26 @@ public void testStartOfNextWeek() {
     Mockito.when(dateUtil.getNow()).thenReturn(setTimeParts(8, false));
     assertEqualStartOfNextWeek(dateUtil, 9);
   }
+
+  // validates logic to display correct dates in MAU chart
+  @Test
+  public void testDateAdjustmentsForMonth() {
+    DateUtil dateUtil = Mockito.spy(DateUtil.class);
+
+    Mockito.when(dateUtil.getNow()).thenReturn(new DateTime(2024, 11, 15, 0, 0, 0));
+
+    // start date should be next month minus a day
+    // but we want to display Dec 1 instead of Nov 30, so add a day and verify it's Dec
+    DateTime startOfNextMonthMinus12 = dateUtil.getStartOfNextMonth().minusMonths(12);
+    DateTime adjustedStart = startOfNextMonthMinus12.minusMillis(1).plusDays(1);
+    assertEquals(12, adjustedStart.getMonthOfYear()); // Verify it is December
+    assertEquals(2023, adjustedStart.getYear()); // Verify it is 2023
+
+    // verify that the end date displays correctly
+    // the chart will display Oct 1 as the last month because we don't show current month
+    DateTime startOfThisMonth = dateUtil.getStartOfThisMonth();
+    DateTime adjustedEnd = startOfThisMonth.minusMillis(1).plusDays(1);
+    assertEquals(11, adjustedEnd.getMonthOfYear()); // Verify it is November
+    assertEquals(2024, adjustedEnd.getYear()); // Verify it is 2024
+  }
 }
diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx
@@ -566,6 +566,12 @@ export const dataset3 = {
                     username: 'datahub',
                     type: EntityType.CorpUser,
                 },
+                actor: {
+                    __typename: 'CorpUser',
+                    urn: 'urn:li:corpuser:datahub',
+                    username: 'datahub',
+                    type: EntityType.CorpUser,
+                },
                 description: 'This only points to Google',
                 label: 'This only points to Google',
                 created: {

diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/LinkButton.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/LinkButton.tsx
@@ -29,7 +29,7 @@ export default function LinkButton({ link }: Props) {
             href={link.url}
             target="_blank"
             rel="noreferrer"
-            key={`${link.label}-${link.url}-${link.author}`}
+            key={`${link.label}-${link.url}-${link.actor.urn}`}
         >
             <LinkOutlined />
             {link.description || link.label}

diff --git a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/LinkList.tsx b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/LinkList.tsx
@@ -3,7 +3,7 @@ import { Link } from 'react-router-dom';
 import styled from 'styled-components/macro';
 import { message, Button, List, Typography, Modal, Form, Input } from 'antd';
 import { LinkOutlined, DeleteOutlined, EditOutlined } from '@ant-design/icons';
-import { EntityType, InstitutionalMemoryMetadata } from '../../../../../../types.generated';
+import { InstitutionalMemoryMetadata } from '../../../../../../types.generated';
 import { useEntityData, useMutationUrn } from '../../../EntityContext';
 import { useEntityRegistry } from '../../../../../useEntityRegistry';
 import { ANTD_GRAY } from '../../../constants';
@@ -182,10 +182,8 @@ export const LinkList = ({ refetch }: LinkListProps) => {
                                 description={
                                     <>
                                         Added {formatDateString(link.created.time)} by{' '}
-                                        <Link
-                                            to={`${entityRegistry.getEntityUrl(EntityType.CorpUser, link.author.urn)}`}
-                                        >
-                                            {link.author.username}
+                                        <Link to={`${entityRegistry.getEntityUrl(link.actor.type, link.actor.urn)}`}>
+                                            {entityRegistry.getDisplayName(link.actor.type, link.actor)}
                                         </Link>
                                     </>
                                 }

diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
@@ -193,7 +193,9 @@ export const IngestionSourceList = () => {
 
     const formatExtraArgs = (extraArgs): StringMapEntryInput[] => {
         if (extraArgs === null || extraArgs === undefined) return [];
-        return extraArgs.map((entry) => ({ key: entry.key, value: entry.value }));
+        return extraArgs
+            .filter((entry) => entry.value !== null && entry.value !== undefined && entry.value !== '')
+            .map((entry) => ({ key: entry.key, value: entry.value }));
     };
 
     const createOrUpdateIngestionSource = (