Merge branch 'datahub-project:master' into master

.github/workflows/airflow-plugin.yml

@@ -40,19 +40,19 @@ jobs:
             extra_pip_requirements: "apache-airflow~=2.2.4"
             extra_pip_extras: plugin-v1
           - python-version: "3.10"
-            extra_pip_requirements: "apache-airflow==2.4.3"
+            extra_pip_requirements: "apache-airflow~=2.4.3"
             extra_pip_extras: plugin-v2,test-airflow24
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow==2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt'
+            extra_pip_requirements: 'apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow==2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt'
+            extra_pip_requirements: 'apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow==2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
+            extra_pip_requirements: 'apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
-          - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow==2.9.0 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.0/constraints-3.10.txt'
+          - python-version: "3.11"
+            extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
       fail-fast: false
     steps:

.github/workflows/documentation.yml

@@ -27,6 +27,8 @@ concurrency:
 jobs:
   gh-pages:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     steps:
       # We explicitly don't use acryldata/sane-checkout-action because docusaurus runs
       # git commands to determine the last change date for each file.

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java

@@ -21,4 +21,5 @@ public class FeatureFlags {
   private boolean schemaFieldEntityFetchEnabled = false;
   private boolean businessAttributeEntityEnabled = false;
   private boolean dataContractsEnabled = false;
+  private boolean showSeparateSiblings = false;
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java

@@ -255,4 +255,19 @@ public static Filter viewFilter(
     Filter result = SearchUtils.combineFilters(null, viewInfo.getDefinition().getFilter());
     return result;
   }
+
+  /**
+   * Simply resolves the end time filter for the search across lineage query. If the start time is
+   * provided, but end time is not provided, we will default to the current time.
+   */
+  public static Long getLineageEndTimeMillis(
+      @Nullable Long startTimeMillis, @Nullable Long endTimeMillis) {
+    if (endTimeMillis != null) {
+      return endTimeMillis;
+    }
+    if (startTimeMillis != null) {
+      return System.currentTimeMillis();
+    }
+    return null;
+  }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java

@@ -186,6 +186,7 @@ public CompletableFuture<AppConfig> get(final DataFetchingEnvironment environmen
             .setNestedDomainsEnabled(_featureFlags.isNestedDomainsEnabled())
             .setPlatformBrowseV2(_featureFlags.isPlatformBrowseV2())
             .setDataContractsEnabled(_featureFlags.isDataContractsEnabled())
+            .setShowSeparateSiblings(_featureFlags.isShowSeparateSiblings())
             .build();
 
     appConfig.setFeatureFlags(featureFlagsConfig);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java

@@ -18,6 +18,7 @@
 import com.linkedin.datahub.graphql.generated.LineageInput;
 import com.linkedin.datahub.graphql.generated.LineageRelationship;
 import com.linkedin.datahub.graphql.generated.Restricted;
+import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
 import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
 import com.linkedin.metadata.graph.SiblingGraphService;
 import graphql.schema.DataFetcher;
@@ -63,7 +64,10 @@ public CompletableFuture<EntityLineageResult> get(DataFetchingEnvironment enviro
     @Nullable final Integer count = input.getCount(); // Optional!
     @Nullable final Boolean separateSiblings = input.getSeparateSiblings(); // Optional!
     @Nullable final Long startTimeMillis = input.getStartTimeMillis(); // Optional!
-    @Nullable final Long endTimeMillis = input.getEndTimeMillis(); // Optional!
+    @Nullable
+    final Long endTimeMillis =
+        ResolverUtils.getLineageEndTimeMillis(
+            input.getStartTimeMillis(), input.getEndTimeMillis()); // Optional!
 
     com.linkedin.metadata.graph.LineageDirection resolvedDirection =
         com.linkedin.metadata.graph.LineageDirection.valueOf(lineageDirection.toString());

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java

@@ -78,7 +78,8 @@ public CompletableFuture<ScrollAcrossLineageResults> get(DataFetchingEnvironment
     @Nullable
     Long startTimeMillis = input.getStartTimeMillis() == null ? null : input.getStartTimeMillis();
     @Nullable
-    Long endTimeMillis = input.getEndTimeMillis() == null ? null : input.getEndTimeMillis();
+    Long endTimeMillis =
+        ResolverUtils.getLineageEndTimeMillis(input.getStartTimeMillis(), input.getEndTimeMillis());
 
     final LineageFlags lineageFlags = LineageFlagsInputMapper.map(context, input.getLineageFlags());
     if (lineageFlags.getStartTimeMillis() == null && startTimeMillis != null) {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java

@@ -111,7 +111,8 @@ public CompletableFuture<SearchAcrossLineageResults> get(DataFetchingEnvironment
     @Nullable
     Long startTimeMillis = input.getStartTimeMillis() == null ? null : input.getStartTimeMillis();
     @Nullable
-    Long endTimeMillis = input.getEndTimeMillis() == null ? null : input.getEndTimeMillis();
+    Long endTimeMillis =
+        ResolverUtils.getLineageEndTimeMillis(input.getStartTimeMillis(), input.getEndTimeMillis());
 
     final LineageFlags lineageFlags = LineageFlagsInputMapper.map(context, input.getLineageFlags());
     if (lineageFlags.getStartTimeMillis() == null && startTimeMillis != null) {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/LineageFlagsInputMapper.java

@@ -6,6 +6,7 @@
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.EntityTypeToPlatforms;
 import com.linkedin.datahub.graphql.generated.LineageFlags;
+import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
 import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper;
 import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
 import java.util.Collections;
@@ -42,12 +43,16 @@ public com.linkedin.metadata.query.LineageFlags apply(
     if (lineageFlags.getIgnoreAsHops() != null) {
       result.setIgnoreAsHops(mapIgnoreAsHops(lineageFlags.getIgnoreAsHops()));
     }
-    if (lineageFlags.getEndTimeMillis() != null) {
-      result.setEndTimeMillis(lineageFlags.getEndTimeMillis());
-    }
     if (lineageFlags.getStartTimeMillis() != null) {
       result.setStartTimeMillis(lineageFlags.getStartTimeMillis());
     }
+    // Default to "now" if no end time is provided, but start time is provided.
+    Long endTimeMillis =
+        ResolverUtils.getLineageEndTimeMillis(
+            lineageFlags.getStartTimeMillis(), lineageFlags.getEndTimeMillis());
+    if (endTimeMillis != null) {
+      result.setEndTimeMillis(endTimeMillis);
+    }
     if (lineageFlags.getEntitiesExploredPerHopLimit() != null) {
       result.setEntitiesExploredPerHopLimit(lineageFlags.getEntitiesExploredPerHopLimit());
     }

datahub-graphql-core/src/main/resources/app.graphql

@@ -497,6 +497,11 @@ type FeatureFlagsConfig {
   Whether data contracts should be enabled
   """
   dataContractsEnabled: Boolean!
+
+  """
+  If turned on, all siblings will be separated with no way to get to a "combined" sibling view
+  """
+  showSeparateSiblings: Boolean!
 }
 
 """

datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx

@@ -12,6 +12,7 @@ import {
     StyledTooltip,
 } from './ParentNodesView';
 import ParentEntities from '../../../../../../search/filters/ParentEntities';
+import { useIsShowSeparateSiblingsEnabled } from '../../../../../../useAppConfig';
 
 export const LogoIcon = styled.span`
     display: flex;
@@ -100,6 +101,10 @@ function PlatformContentView(props: Props) {
     const directParentContainer = parentContainers && parentContainers[0];
     const remainingParentContainers = parentContainers && parentContainers.slice(1, parentContainers.length);
 
+    const shouldShowSeparateSiblings = useIsShowSeparateSiblingsEnabled();
+    const showSiblingPlatformLogos = !shouldShowSeparateSiblings && !!platformLogoUrls;
+    const showSiblingPlatformNames = !shouldShowSeparateSiblings && !!platformNames;
+
     return (
         <PlatformContentWrapper>
             {typeIcon && <LogoIcon>{typeIcon}</LogoIcon>}
@@ -110,10 +115,10 @@ function PlatformContentView(props: Props) {
             {platformName && (
                 <LogoIcon>
                     {!platformLogoUrl && !platformLogoUrls && entityLogoComponent}
-                    {!!platformLogoUrl && !platformLogoUrls && (
+                    {!!platformLogoUrl && !showSiblingPlatformLogos && (
                         <PreviewImage preview={false} src={platformLogoUrl} alt={platformName} />
                     )}
-                    {!!platformLogoUrls &&
+                    {showSiblingPlatformLogos &&
                         platformLogoUrls.slice(0, 2).map((platformLogoUrlsEntry) => (
                             <>
                                 <PreviewImage preview={false} src={platformLogoUrlsEntry || ''} alt={platformName} />
@@ -122,7 +127,7 @@ function PlatformContentView(props: Props) {
                 </LogoIcon>
             )}
             <PlatformText>
-                {platformNames ? platformNames.join(' & ') : platformName}
+                {showSiblingPlatformNames ? platformNames.join(' & ') : platformName}
                 {(directParentContainer || instanceId) && <StyledRightOutlined data-testid="right-arrow" />}
             </PlatformText>
             {instanceId && (

datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarSiblingsSection.tsx

@@ -7,6 +7,7 @@ import { CompactEntityNameList } from '../../../../../recommendations/renderer/c
 import { Dataset, Entity } from '../../../../../../types.generated';
 import { SEPARATE_SIBLINGS_URL_PARAM, stripSiblingsFromEntity, useIsSeparateSiblingsMode } from '../../../siblingUtils';
 import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated';
+import { useIsShowSeparateSiblingsEnabled } from '../../../../../useAppConfig';
 
 const EntityListContainer = styled.div`
     margin-left: -8px;
@@ -16,13 +17,15 @@ export const SidebarSiblingsSection = () => {
     const { entityData } = useEntityData();
     const dataNotCombinedWithSiblings = useDataNotCombinedWithSiblings<GetDatasetQuery>();
 
+    const showSeparateSiblings = useIsShowSeparateSiblingsEnabled();
     const isHideSiblingMode = useIsSeparateSiblingsMode();
 
     if (!entityData) {
         return <></>;
     }
 
-    if (isHideSiblingMode) {
+    // showSeparateSiblings disables the combined view, but with this flag on we show siblings in the sidebar to navigate to them
+    if (!showSeparateSiblings && isHideSiblingMode) {
         return (
             <div>
                 <SidebarHeader title="Part Of" />
@@ -36,13 +39,19 @@ export const SidebarSiblingsSection = () => {
     const siblingEntities = entityData?.siblings?.siblings || [];
     const entityDataWithoutSiblings = stripSiblingsFromEntity(dataNotCombinedWithSiblings.dataset);
 
-    const allSiblingsInGroup = [...siblingEntities, entityDataWithoutSiblings] as Dataset[];
+    const allSiblingsInGroup = showSeparateSiblings
+        ? (siblingEntities as Dataset[])
+        : ([...siblingEntities, entityDataWithoutSiblings] as Dataset[]);
 
     const allSiblingsInGroupThatExist = allSiblingsInGroup.filter((sibling) => sibling.exists);
 
+    if (!allSiblingsInGroupThatExist.length) {
+        return <></>;
+    }
+
     // you are always going to be in the sibling group, so if the sibling group is just you do not render.
     // The less than case is likely not neccessary but just there as a safety case for unexpected scenarios
-    if (allSiblingsInGroupThatExist.length <= 1) {
+    if (!showSeparateSiblings && allSiblingsInGroupThatExist.length <= 1) {
         return <></>;
     }
 

datahub-web-react/src/app/entity/shared/siblingUtils.ts

@@ -4,6 +4,7 @@ import { useLocation } from 'react-router-dom';
 import * as QueryString from 'query-string';
 import { Dataset, Entity, Maybe, SiblingProperties } from '../../../types.generated';
 import { GenericEntityProperties } from './types';
+import { useIsShowSeparateSiblingsEnabled } from '../../useAppConfig';
 
 export function stripSiblingsFromEntity(entity: any) {
     return {
@@ -267,8 +268,9 @@ export const SEPARATE_SIBLINGS_URL_PARAM = 'separate_siblings';
 
 // used to determine whether sibling entities should be shown merged or not
 export function useIsSeparateSiblingsMode() {
+    const showSeparateSiblings = useIsShowSeparateSiblingsEnabled();
     const location = useLocation();
     const params = QueryString.parse(location.search, { arrayFormat: 'comma' });
 
-    return params[SEPARATE_SIBLINGS_URL_PARAM] === 'true';
+    return showSeparateSiblings || params[SEPARATE_SIBLINGS_URL_PARAM] === 'true';
 }

datahub-web-react/src/app/ingest/ManageIngestionPage.tsx

@@ -2,6 +2,8 @@ import { Tabs, Typography } from 'antd';
 import React, { useState } from 'react';
 import styled from 'styled-components';
 import { IngestionSourceList } from './source/IngestionSourceList';
+import { useAppConfig } from '../useAppConfig';
+import { useUserContext } from '../context/useUserContext';
 import { SecretsList } from './secret/SecretsList';
 import { OnboardingTour } from '../onboarding/OnboardingTour';
 import {
@@ -48,7 +50,13 @@ export const ManageIngestionPage = () => {
     /**
      * Determines which view should be visible: ingestion sources or secrets.
      */
-    const [selectedTab, setSelectedTab] = useState<TabType>(TabType.Sources);
+    const me = useUserContext();
+    const { config } = useAppConfig();
+    const isIngestionEnabled = config?.managedIngestionConfig.enabled;
+    const showIngestionTab = isIngestionEnabled && me && me.platformPrivileges?.manageIngestion;
+    const showSecretsTab = isIngestionEnabled && me && me.platformPrivileges?.manageSecrets;
+    const defaultTab = showIngestionTab ? TabType.Sources : TabType.Secrets;
+    const [selectedTab, setSelectedTab] = useState<TabType>(defaultTab);
 
     const onClickTab = (newTab: string) => {
         setSelectedTab(TabType[newTab]);
@@ -64,8 +72,8 @@ export const ManageIngestionPage = () => {
                 </Typography.Paragraph>
             </PageHeaderContainer>
             <StyledTabs activeKey={selectedTab} size="large" onTabClick={(tab: string) => onClickTab(tab)}>
-                <Tab key={TabType.Sources} tab={TabType.Sources} />
-                <Tab key={TabType.Secrets} tab={TabType.Secrets} />
+                {showIngestionTab && <Tab key={TabType.Sources} tab={TabType.Sources} />}
+                {showSecretsTab && <Tab key={TabType.Secrets} tab={TabType.Secrets} />}
             </StyledTabs>
             <ListContainer>{selectedTab === TabType.Sources ? <IngestionSourceList /> : <SecretsList />}</ListContainer>
         </PageContainer>

datahub-web-react/src/app/ingest/secret/SecretsList.tsx

@@ -118,7 +118,7 @@ export const SecretsList = () => {
                     {
                         urn: res.data?.createSecret || '',
                         name: state.name,
-                        description: state.description,
+                        description: state.description || '',
                     },
                     client,
                     pageSize,
@@ -127,7 +127,7 @@ export const SecretsList = () => {
             .catch((e) => {
                 message.destroy();
                 message.error({
-                    content: `Failed to update ingestion source!: \n ${e.message || ''}`,
+                    content: `Failed to update secret!: \n ${e.message || ''}`,
                     duration: 3,
                 });
             });

datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx

@@ -32,6 +32,7 @@ import {
     INGESTION_REFRESH_SOURCES_ID,
 } from '../../onboarding/config/IngestionOnboardingConfig';
 import { ONE_SECOND_IN_MS } from '../../entity/shared/tabs/Dataset/Queries/utils/constants';
+import { useCommandS } from './hooks';
 
 const PLACEHOLDER_URN = 'placeholder-urn';
 
@@ -51,6 +52,8 @@ const FilterWrapper = styled.div`
     display: flex;
 `;
 
+const SYSTEM_INTERNAL_SOURCE_TYPE = 'SYSTEM';
+
 export enum IngestionSourceType {
     ALL,
     UI,
@@ -102,6 +105,17 @@ export const IngestionSourceList = () => {
     // Set of removed urns used to account for eventual consistency
     const [removedUrns, setRemovedUrns] = useState<string[]>([]);
     const [sourceFilter, setSourceFilter] = useState(IngestionSourceType.ALL);
+    const [hideSystemSources, setHideSystemSources] = useState(true);
+
+    /**
+     * Show or hide system ingestion sources using a hidden command S command.
+     */
+    useCommandS(() => setHideSystemSources(!hideSystemSources));
+
+    // Ingestion Source Default Filters
+    const filters = hideSystemSources
+        ? [{ field: 'sourceType', values: [SYSTEM_INTERNAL_SOURCE_TYPE], negated: true }]
+        : undefined;
 
     // Ingestion Source Queries
     const { loading, error, data, client, refetch } = useListIngestionSourcesQuery({
@@ -110,6 +124,7 @@ export const IngestionSourceList = () => {
                 start,
                 count: pageSize,
                 query: (query?.length && query) || undefined,
+                filters,
             },
         },
         fetchPolicy: (query?.length || 0) > 0 ? 'no-cache' : 'cache-first',

datahub-web-react/src/app/ingest/source/hooks.ts

@@ -0,0 +1,16 @@
+import { useEffect } from 'react';
+
+export const useCommandS = (onPress: () => void) => {
+    useEffect(() => {
+        const handleKeyDown = (event: KeyboardEvent) => {
+            if (event.metaKey && event.key === 's') {
+                event.preventDefault();
+                onPress();
+            }
+        };
+        window.addEventListener('keydown', handleKeyDown);
+        return () => {
+            window.removeEventListener('keydown', handleKeyDown);
+        };
+    }, [onPress]);
+};