From 18a6c7fa08efff86b42e0cecf710c080c300c4dc Mon Sep 17 00:00:00 2001 From: aditya-radhakrishnan Date: Mon, 5 Dec 2022 18:55:28 -0800 Subject: [PATCH] perf(ui-ingestion): cache on creation or deletion of ingestion sources to reduce latency --- .../app/ingest/source/IngestionSourceList.tsx | 16 +++- .../ingest/source/IngestionSourceTable.tsx | 2 +- .../src/app/ingest/source/utils.ts | 83 +++++++++++++++++++ 3 files changed, 98 insertions(+), 3 deletions(-) diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx index 2700a53bd19177..4429a0e91b2abb 100644 --- a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx +++ b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx @@ -14,7 +14,7 @@ import { import { Message } from '../../shared/Message'; import TabToolbar from '../../entity/shared/components/styled/TabToolbar'; import { IngestionSourceBuilderModal } from './builder/IngestionSourceBuilderModal'; -import { CLI_EXECUTOR_ID } from './utils'; +import { addToListIngestionSourcesCache, CLI_EXECUTOR_ID, removeFromListIngestionSourcesCache } from './utils'; import { DEFAULT_EXECUTOR_ID, SourceBuilderState } from './builder/types'; import { IngestionSource, UpdateIngestionSourceInput } from '../../../types.generated'; import { SearchBar } from '../../search/SearchBar'; @@ -96,7 +96,7 @@ export const IngestionSourceList = () => { const [sourceFilter, setSourceFilter] = useState(IngestionSourceType.ALL); // Ingestion Source Queries - const { loading, error, data, refetch } = useListIngestionSourcesQuery({ + const { loading, error, data, client, refetch } = useListIngestionSourcesQuery({ variables: { input: { start, @@ -104,6 +104,7 @@ export const IngestionSourceList = () => { query, }, }, + fetchPolicy: 'cache-first', }); const [createIngestionSource] = useCreateIngestionSourceMutation(); const [updateIngestionSource] = useUpdateIngestionSourceMutation(); @@ -197,6 +198,16 @@ export const IngestionSourceList = () => { }); } else { // Create + const newSource = { + urn: 'placeholder-urn', + name: input.name, + type: input.type, + config: null, + schedule: null, + platform: null, + executions: null, + }; + addToListIngestionSourcesCache(client, newSource, pageSize, query); createIngestionSource({ variables: { input } }) .then((result) => { message.loading({ content: 'Loading...', duration: 2 }); @@ -236,6 +247,7 @@ export const IngestionSourceList = () => { }; const deleteIngestionSource = async (urn: string) => { + removeFromListIngestionSourcesCache(client, urn, page, pageSize, query); removeIngestionSourceMutation({ variables: { urn }, }) diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceTable.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceTable.tsx index bfafb81538bbbe..9c2c9f13f73a54 100644 --- a/datahub-web-react/src/app/ingest/source/IngestionSourceTable.tsx +++ b/datahub-web-react/src/app/ingest/source/IngestionSourceTable.tsx @@ -122,7 +122,7 @@ function IngestionSourceTable({ source.executions?.total && source.executions?.total > 0 && source.executions?.executionRequests[0].result?.status, - cliIngestion: source.config.executorId === CLI_EXECUTOR_ID, + cliIngestion: source.config?.executorId === CLI_EXECUTOR_ID, })); return ( diff --git a/datahub-web-react/src/app/ingest/source/utils.ts b/datahub-web-react/src/app/ingest/source/utils.ts index b2f20c93614e2a..c372388e958b78 100644 --- a/datahub-web-react/src/app/ingest/source/utils.ts +++ b/datahub-web-react/src/app/ingest/source/utils.ts @@ -11,6 +11,7 @@ import { EntityType, FacetMetadata } from '../../../types.generated'; import { capitalizeFirstLetterOnly, pluralize } from '../../shared/textUtil'; import EntityRegistry from '../../entity/EntityRegistry'; import { SourceConfig } from './builder/types'; +import { ListIngestionSourcesDocument, ListIngestionSourcesQuery } from '../../../graphql/ingestion.generated'; export const getSourceConfigs = (ingestionSources: SourceConfig[], sourceType: string) => { const sourceConfigs = ingestionSources.find((source) => source.name === sourceType); @@ -171,3 +172,85 @@ export const extractEntityTypeCountsFromFacets = ( return finalCounts; }; + +/** + * Add an entry to the ListIngestionSources cache. + */ +export const addToListIngestionSourcesCache = (client, newSource, pageSize, query) => { + // Read the data from our cache for this query. + const currData: ListIngestionSourcesQuery | null = client.readQuery({ + query: ListIngestionSourcesDocument, + variables: { + input: { + start: 0, + count: pageSize, + query, + }, + }, + }); + + // Add our new source into the existing list. + const newSources = [newSource, ...(currData?.listIngestionSources?.ingestionSources || [])]; + + // Write our data back to the cache. + client.writeQuery({ + query: ListIngestionSourcesDocument, + variables: { + input: { + start: 0, + count: pageSize, + query, + }, + }, + data: { + listIngestionSources: { + start: 0, + count: (currData?.listIngestionSources?.count || 0) + 1, + total: (currData?.listIngestionSources?.total || 0) + 1, + ingestionSources: newSources, + }, + }, + }); +}; + +/** + * Remove an entry from the ListIngestionSources cache. + */ +export const removeFromListIngestionSourcesCache = (client, urn, page, pageSize, query) => { + // Read the data from our cache for this query. + const currData: ListIngestionSourcesQuery | null = client.readQuery({ + query: ListIngestionSourcesDocument, + variables: { + input: { + start: (page - 1) * pageSize, + count: pageSize, + query, + }, + }, + }); + + // Remove the source from the existing sources set. + const newSources = [ + ...(currData?.listIngestionSources?.ingestionSources || []).filter((source) => source.urn !== urn), + ]; + + // Write our data back to the cache. + client.writeQuery({ + query: ListIngestionSourcesDocument, + variables: { + input: { + start: (page - 1) * pageSize, + count: pageSize, + query, + }, + }, + data: { + listIngestionSources: { + start: currData?.listIngestionSources?.start || 0, + count: (currData?.listIngestionSources?.count || 1) - 1, + total: (currData?.listIngestionSources?.total || 1) - 1, + ingestionSources: newSources, + }, + }, + }); +};