Merge Master 18.11.2022.

.github/workflows/metadata-ingestion.yml

@@ -38,6 +38,11 @@ jobs:
             "testIntegrationBatch1",
             "testSlowIntegration",
           ]
+        include:
+          - python-version: "3.7"
+            extraPythonRequirement: "sqlalchemy==1.3.24 apache-airflow~=2.2.0"
+          - python-version: "3.10"
+            extraPythonRequirement: "sqlalchemy~=1.4.0 apache-airflow>=2.4.0"
       fail-fast: false
     steps:
       - uses: actions/checkout@v3
@@ -50,8 +55,8 @@ jobs:
           hadoop-version: "3.2"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
-      - name: Run metadata-ingestion tests
-        run: ./gradlew :metadata-ingestion:build :metadata-ingestion:${{ matrix.command }}
+      - name: Run metadata-ingestion tests (extras ${{ matrix.extraPythonRequirement }})
+        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion:${{ matrix.command }}
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion/venv/bin/activate && pip freeze

build.gradle

@@ -149,6 +149,7 @@ project.ext.externalDependency = [
     'springJdbc': "org.springframework:spring-jdbc:$springVersion",
     'springWeb': "org.springframework:spring-web:$springVersion",
     'springWebMVC': "org.springframework:spring-webmvc:$springVersion",
+    'springBootTest': "org.springframework.boot:spring-boot-starter-test:$springBootVersion",
     'springBoot': "org.springframework.boot:spring-boot:$springBootVersion",
     'springBootAutoconfigure': "org.springframework.boot:spring-boot-autoconfigure:$springBootVersion",
     'springBootStarterWeb': "org.springframework.boot:spring-boot-starter-web:$springBootVersion",

datahub-frontend/conf/logback.xml

@@ -5,7 +5,7 @@
 
   <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
     <layout class="ch.qos.logback.classic.PatternLayout">
-      <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
+      <pattern>%date{ISO8601} [%thread] %-5level %logger{36} - %msg%n</pattern>
     </layout>
     <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
       <level>INFO</level>
@@ -27,7 +27,7 @@
       <level>INFO</level>
     </filter>
     <encoder>
-      <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
+      <pattern>%date{ISO8601} [%thread] %-5level %logger{36} - %msg%n</pattern>
     </encoder>
   </appender>
 
@@ -48,7 +48,7 @@
       <onMismatch>DENY</onMismatch>
     </filter>
     <encoder>
-      <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
+      <pattern>%date{ISO8601} [%thread] %-5level %logger{36} - %msg%n</pattern>
     </encoder>
   </appender>
 

datahub-frontend/run/logback.xml

@@ -5,7 +5,7 @@
 
   <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
     <layout class="ch.qos.logback.classic.PatternLayout">
-      <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
+      <pattern>%date{ISO8601} [%thread] %-5level %logger{36} - %msg%n</pattern>
     </layout>
     <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
       <level>INFO</level>
@@ -27,7 +27,7 @@
       <level>INFO</level>
     </filter>
     <encoder>
-      <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
+      <pattern>%date{ISO8601} [%thread] %-5level %logger{36} - %msg%n</pattern>
     </encoder>
   </appender>
 
@@ -48,7 +48,7 @@
       <onMismatch>DENY</onMismatch>
     </filter>
     <encoder>
-      <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
+      <pattern>%date{ISO8601} [%thread] %-5level %logger{36} - %msg%n</pattern>
     </encoder>
   </appender>
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/role/AcceptRoleResolver.java

@@ -8,6 +8,7 @@
 import com.linkedin.datahub.graphql.generated.AcceptRoleInput;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
+import java.util.Collections;
 import java.util.concurrent.CompletableFuture;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -32,15 +33,14 @@ public CompletableFuture<Boolean> get(DataFetchingEnvironment environment) throw
 
     return CompletableFuture.supplyAsync(() -> {
       try {
-        Urn inviteTokenUrn = _inviteTokenService.getInviteTokenUrn(inviteTokenStr);
+        final Urn inviteTokenUrn = _inviteTokenService.getInviteTokenUrn(inviteTokenStr);
         if (!_inviteTokenService.isInviteTokenValid(inviteTokenUrn, authentication)) {
           throw new RuntimeException(String.format("Invite token %s is invalid", inviteTokenStr));
         }
 
-        Urn roleUrn = _inviteTokenService.getInviteTokenRole(inviteTokenUrn, authentication);
-        if (roleUrn != null) {
-          _roleService.assignRoleToActor(authentication.getActor().toUrnStr(), roleUrn, authentication);
-        }
+        final Urn roleUrn = _inviteTokenService.getInviteTokenRole(inviteTokenUrn, authentication);
+        _roleService.batchAssignRoleToActors(Collections.singletonList(authentication.getActor().toUrnStr()), roleUrn,
+            authentication);
 
         return true;
       } catch (Exception e) {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/role/BatchAssignRoleResolver.java

@@ -37,19 +37,8 @@ public CompletableFuture<Boolean> get(DataFetchingEnvironment environment) throw
 
     return CompletableFuture.supplyAsync(() -> {
       try {
-        Urn roleUrn = Urn.createFromString(roleUrnStr);
-        if (!_roleService.exists(roleUrn, authentication)) {
-          throw new RuntimeException(String.format("Role %s does not exist", roleUrnStr));
-        }
-
-        actors.forEach(actor -> {
-          try {
-            _roleService.assignRoleToActor(actor, roleUrn, authentication);
-          } catch (Exception e) {
-            log.warn(
-                String.format("Failed to assign role %s to actor %s. Skipping actor assignment", roleUrnStr, actor), e);
-          }
-        });
+        final Urn roleUrn = roleUrnStr == null ? null : Urn.createFromString(roleUrnStr);
+        _roleService.batchAssignRoleToActors(actors, roleUrn, authentication);
         return true;
       } catch (Exception e) {
         throw new RuntimeException(String.format("Failed to perform update against input %s", input), e);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/role/ListRolesResolver.java

@@ -2,7 +2,6 @@
 
 import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.graphql.QueryContext;
-import com.linkedin.datahub.graphql.exception.AuthorizationException;
 import com.linkedin.datahub.graphql.generated.DataHubRole;
 import com.linkedin.datahub.graphql.generated.ListRolesInput;
 import com.linkedin.datahub.graphql.generated.ListRolesResult;
@@ -24,7 +23,6 @@
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
-import static com.linkedin.datahub.graphql.authorization.AuthorizationUtils.*;
 import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*;
 import static com.linkedin.metadata.Constants.*;
 
@@ -41,10 +39,6 @@ public class ListRolesResolver implements DataFetcher<CompletableFuture<ListRole
   @Override
   public CompletableFuture<ListRolesResult> get(final DataFetchingEnvironment environment) throws Exception {
     final QueryContext context = environment.getContext();
-    if (!canManagePolicies(context)) {
-      throw new AuthorizationException(
-          "Unauthorized to view roles. Please contact your DataHub administrator if this needs corrective action.");
-    }
 
     final ListRolesInput input = bindArgument(environment.getArgument("input"), ListRolesInput.class);
     final Integer start = input.getStart() == null ? DEFAULT_START : input.getStart();

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapper.java

@@ -23,6 +23,9 @@ public MLModelProperties apply(@NonNull final com.linkedin.ml.metadata.MLModelPr
 
         result.setDate(mlModelProperties.getDate());
         result.setDescription(mlModelProperties.getDescription());
+        if (mlModelProperties.getExternalUrl() != null) {
+            result.setExternalUrl(mlModelProperties.getExternalUrl().toString());
+        }        
         if (mlModelProperties.getVersion() != null) {
             result.setVersion(mlModelProperties.getVersion().getVersionTag());
         }

datahub-graphql-core/src/main/resources/entity.graphql

@@ -8333,6 +8333,8 @@ type MLModelProperties {
     groups: [MLModelGroup]
 
     customProperties: [CustomPropertiesEntry!]
+
+    externalUrl: String
 }
 
 type MLFeatureProperties {
@@ -9326,9 +9328,9 @@ Input provided when batch assigning a role to a list of users
 """
 input BatchAssignRoleInput {
     """
-    The urn of the role to assign to the actors
+    The urn of the role to assign to the actors. If undefined, will remove the role.
     """
-    roleUrn: String!
+    roleUrn: String
 
     """
     The urns of the actors to assign the role to

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/role/AcceptRoleResolverTest.java

@@ -71,13 +71,16 @@ public void testNoRoleUrn() throws Exception {
     when(_inviteTokenService.getInviteTokenUrn(eq(INVITE_TOKEN_STRING))).thenReturn(inviteTokenUrn);
     when(_inviteTokenService.isInviteTokenValid(eq(inviteTokenUrn), eq(_authentication))).thenReturn(true);
     when(_inviteTokenService.getInviteTokenRole(eq(inviteTokenUrn), eq(_authentication))).thenReturn(null);
+    Actor actor = mock(Actor.class);
+    when(_authentication.getActor()).thenReturn(actor);
+    when(actor.toUrnStr()).thenReturn(ACTOR_URN_STRING);
 
     AcceptRoleInput input = new AcceptRoleInput();
     input.setInviteToken(INVITE_TOKEN_STRING);
     when(_dataFetchingEnvironment.getArgument(eq("input"))).thenReturn(input);
 
     assertTrue(_resolver.get(_dataFetchingEnvironment).join());
-    verify(_roleService, never()).assignRoleToActor(any(), any(), any());
+    verify(_roleService, times(1)).batchAssignRoleToActors(any(), any(), any());
   }
 
   @Test
@@ -97,6 +100,6 @@ public void testAssignRolePasses() throws Exception {
     when(_dataFetchingEnvironment.getArgument(eq("input"))).thenReturn(input);
 
     assertTrue(_resolver.get(_dataFetchingEnvironment).join());
-    verify(_roleService, times(1)).assignRoleToActor(any(), any(), any());
+    verify(_roleService, times(1)).batchAssignRoleToActors(any(), any(), any());
   }
 }

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/role/BatchAssignRoleResolverTest.java

@@ -2,11 +2,11 @@
 
 import com.datahub.authentication.Authentication;
 import com.datahub.authorization.role.RoleService;
+import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.BatchAssignRoleInput;
 import graphql.schema.DataFetchingEnvironment;
-import java.util.ArrayList;
 import java.util.List;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
@@ -45,56 +45,30 @@ public void testNotAuthorizedFails() {
   }
 
   @Test
-  public void testRoleDoesNotExistFails() throws Exception {
+  public void testNullRole() throws Exception {
     QueryContext mockContext = getMockAllowContext();
     when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext);
     when(mockContext.getAuthentication()).thenReturn(_authentication);
 
     BatchAssignRoleInput input = new BatchAssignRoleInput();
-    input.setRoleUrn(ROLE_URN_STRING);
-    List<String> actors = new ArrayList<>();
-    actors.add(FIRST_ACTOR_URN_STRING);
-    input.setActors(actors);
-    when(_dataFetchingEnvironment.getArgument("input")).thenReturn(input);
-    when(_roleService.exists(eq(roleUrn), eq(_authentication))).thenReturn(false);
-
-    assertThrows(() -> _resolver.get(_dataFetchingEnvironment).join());
-  }
-
-  @Test
-  public void testSomeActorsExist() throws Exception {
-    QueryContext mockContext = getMockAllowContext();
-    when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext);
-    when(mockContext.getAuthentication()).thenReturn(_authentication);
-
-    BatchAssignRoleInput input = new BatchAssignRoleInput();
-    input.setRoleUrn(ROLE_URN_STRING);
-    List<String> actors = new ArrayList<>();
-    actors.add(FIRST_ACTOR_URN_STRING);
-    actors.add(SECOND_ACTOR_URN_STRING);
+    List<String> actors = ImmutableList.of(FIRST_ACTOR_URN_STRING, SECOND_ACTOR_URN_STRING);
     input.setActors(actors);
     when(_dataFetchingEnvironment.getArgument(eq("input"))).thenReturn(input);
-    doThrow(RuntimeException.class).when(_roleService)
-        .assignRoleToActor(eq(SECOND_ACTOR_URN_STRING), eq(roleUrn), eq(_authentication));
-    when(_roleService.exists(eq(Urn.createFromString(ROLE_URN_STRING)), eq(_authentication))).thenReturn(true);
 
     assertTrue(_resolver.get(_dataFetchingEnvironment).join());
   }
 
   @Test
-  public void testAllActorsExist() throws Exception {
+  public void testNotNullRole() throws Exception {
     QueryContext mockContext = getMockAllowContext();
     when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext);
     when(mockContext.getAuthentication()).thenReturn(_authentication);
 
     BatchAssignRoleInput input = new BatchAssignRoleInput();
     input.setRoleUrn(ROLE_URN_STRING);
-    List<String> actors = new ArrayList<>();
-    actors.add(FIRST_ACTOR_URN_STRING);
-    actors.add(SECOND_ACTOR_URN_STRING);
+    List<String> actors = ImmutableList.of(FIRST_ACTOR_URN_STRING, SECOND_ACTOR_URN_STRING);
     input.setActors(actors);
     when(_dataFetchingEnvironment.getArgument(eq("input"))).thenReturn(input);
-    when(_roleService.exists(eq(Urn.createFromString(ROLE_URN_STRING)), eq(_authentication))).thenReturn(true);
 
     assertTrue(_resolver.get(_dataFetchingEnvironment).join());
   }

datahub-web-react/package.json

@@ -66,7 +66,7 @@
         "js-cookie": "^2.2.1",
         "lodash.debounce": "^4.0.8",
         "miragejs": "^0.1.41",
-        "moment-timezone": "^0.5.34",
+        "moment-timezone": "^0.5.35",
         "monaco-editor": "^0.28.1",
         "monaco-yaml": "^3.2.1",
         "query-string": "^6.13.8",
@@ -89,6 +89,7 @@
         "styled-components": "^5.2.1",
         "typescript": "^4.1.3",
         "uuid": "^8.3.2",
+        "virtualizedtableforantd4": "^1.2.1",
         "vx": "^0.0.1",
         "web-vitals": "^0.2.4",
         "yamljs": "^0.3.0"

datahub-web-react/src/app/analytics/event.ts

@@ -22,6 +22,7 @@ export enum EventType {
     RecommendationImpressionEvent,
     RecommendationClickEvent,
     HomePageRecommendationClickEvent,
+    HomePageExploreAllClickEvent,
     SearchAcrossLineageEvent,
     SearchAcrossLineageResultsViewEvent,
     DownloadAsCsvEvent,
@@ -338,6 +339,10 @@ export interface ShowStandardHomepageEvent extends BaseEvent {
     type: EventType.ShowStandardHomepageEvent;
 }
 
+export interface HomePageExploreAllClickEvent extends BaseEvent {
+    type: EventType.HomePageExploreAllClickEvent;
+}
+
 // Business glossary events
 
 export interface CreateGlossaryEntityEvent extends BaseEvent {
@@ -389,6 +394,7 @@ export type Event =
     | ResetCredentialsEvent
     | SearchEvent
     | HomePageSearchEvent
+    | HomePageExploreAllClickEvent
     | SearchResultsViewEvent
     | SearchResultClickEvent
     | BrowseResultClickEvent

datahub-web-react/src/app/entity/dataset/profile/__tests__/Schema.test.tsx

@@ -15,6 +15,15 @@ import { SchemaTab } from '../../../shared/tabs/Dataset/Schema/SchemaTab';
 import EntityContext from '../../../shared/EntityContext';
 import { EntityType, SchemaMetadata } from '../../../../../types.generated';
 
+jest.mock('virtualizedtableforantd4', () => {
+    /* eslint-disable-next-line */
+    const { SchemaRow } = require('../../../shared/tabs/Dataset/Schema/components/SchemaRow');
+    return {
+        ...jest.requireActual('virtualizedtableforantd4'),
+        useVT: () => [{ body: { row: SchemaRow } }, jest.fn()],
+    };
+});
+
 describe('Schema', () => {
     it('renders', () => {
         const { getByText } = render(

datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx

@@ -57,8 +57,10 @@ export class MLModelEntity implements Entity<MlModel> {
 
     getCollectionName = () => 'ML Models';
 
-    getOverridePropertiesFromEntity = (_?: MlModel | null): GenericEntityProperties => {
-        return {};
+    getOverridePropertiesFromEntity = (mlModel?: MlModel | null): GenericEntityProperties => {
+        return {
+            externalUrl: mlModel?.properties?.externalUrl,
+        };
     };
 
     renderProfile = (urn: string) => (

datahub-web-react/src/app/entity/shared/components/styled/StyledTable.tsx

@@ -1,8 +1,11 @@
-import styled from 'styled-components';
 import { Table } from 'antd';
+import styled from 'styled-components';
 import { ANTD_GRAY } from '../../constants';
 
 export const StyledTable = styled(Table)`
+    overflow: inherit;
+    height: inherit;
+
     &&& .ant-table-cell {
         background-color: #fff;
     }