diff --git a/.gitignore b/.gitignore index bb25c9979e..f352c3dae5 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,5 @@ dist/ coverage.xml .coverage.* pyvenv.cfg + +.env \ No newline at end of file diff --git a/app/packages/app/src/pages/datasets/__generated__/DatasetPageQuery.graphql.ts b/app/packages/app/src/pages/datasets/__generated__/DatasetPageQuery.graphql.ts index f4e92d4510..a17b25a05c 100644 --- a/app/packages/app/src/pages/datasets/__generated__/DatasetPageQuery.graphql.ts +++ b/app/packages/app/src/pages/datasets/__generated__/DatasetPageQuery.graphql.ts @@ -1,5 +1,5 @@ /** - * @generated SignedSource<> + * @generated SignedSource<> * @lightSyntaxTransform * @nogrep */ @@ -847,6 +847,31 @@ return { "plural": false, "selections": [ (v24/*: any*/), + { + "alias": null, + "args": null, + "concreteType": "FieldVisibilityConfig", + "kind": "LinkedField", + "name": "defaultVisibilityLabels", + "plural": false, + "selections": [ + { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "include", + "storageKey": null + }, + { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "exclude", + "storageKey": null + } + ], + "storageKey": null + }, (v25/*: any*/), { "alias": null, @@ -1452,12 +1477,12 @@ return { ] }, "params": { - "cacheID": "f99a6a7bf7511890fdd5db148152135a", + "cacheID": "6075b4fbbfbf557c858ae76d346cc506", "id": null, "metadata": {}, "name": "DatasetPageQuery", "operationKind": "query", - "text": "query DatasetPageQuery(\n $count: Int\n $cursor: String\n $name: String!\n $extendedView: BSONArray!\n $savedViewSlug: String\n $search: String = \"\"\n $view: BSONArray!\n) {\n config {\n colorBy\n colorPool\n colorscale\n multicolorKeypoints\n showSkeletons\n }\n colorscale\n dataset(name: $name, view: $extendedView, savedViewSlug: $savedViewSlug) {\n name\n defaultGroupSlice\n appConfig {\n colorScheme {\n id\n colorBy\n colorPool\n multicolorKeypoints\n opacity\n showSkeletons\n defaultMaskTargetsColors {\n intTarget\n color\n }\n defaultColorscale {\n name\n list {\n value\n color\n }\n rgb\n }\n colorscales {\n path\n name\n list {\n value\n color\n }\n rgb\n }\n fields {\n colorByAttribute\n fieldColor\n path\n valueColors {\n color\n value\n }\n maskTargetsColors {\n intTarget\n color\n }\n }\n labelTags {\n fieldColor\n valueColors {\n color\n value\n }\n }\n }\n }\n ...datasetFragment\n id\n }\n ...NavFragment\n ...savedViewsFragment\n ...configFragment\n ...stageDefinitionsFragment\n ...viewSchemaFragment\n}\n\nfragment Analytics on Query {\n context\n dev\n doNotTrack\n uid\n version\n}\n\nfragment NavDatasets on Query {\n datasets(search: $search, first: $count, after: $cursor) {\n total\n edges {\n cursor\n node {\n name\n id\n __typename\n }\n }\n pageInfo {\n endCursor\n hasNextPage\n }\n }\n}\n\nfragment NavFragment on Query {\n ...Analytics\n ...NavDatasets\n}\n\nfragment colorSchemeFragment on ColorScheme {\n id\n colorBy\n colorPool\n multicolorKeypoints\n opacity\n showSkeletons\n labelTags {\n fieldColor\n valueColors {\n color\n value\n }\n }\n defaultMaskTargetsColors {\n intTarget\n color\n }\n defaultColorscale {\n name\n list {\n value\n color\n }\n rgb\n }\n colorscales {\n path\n name\n list {\n value\n color\n }\n rgb\n }\n fields {\n colorByAttribute\n fieldColor\n path\n valueColors {\n color\n value\n }\n maskTargetsColors {\n intTarget\n color\n }\n }\n}\n\nfragment configFragment on Query {\n config {\n colorBy\n colorPool\n colorscale\n disableFrameFiltering\n gridZoom\n enableQueryPerformance\n defaultQueryPerformance\n loopVideos\n mediaFallback\n multicolorKeypoints\n notebookHeight\n plugins\n showConfidence\n showIndex\n showLabel\n showSkeletons\n showTooltip\n theme\n timezone\n useFrameNumber\n }\n colorscale\n}\n\nfragment datasetAppConfigFragment on DatasetAppConfig {\n colorScheme {\n ...colorSchemeFragment\n id\n }\n disableFrameFiltering\n dynamicGroupsTargetFrameRate\n gridMediaField\n mediaFields\n modalMediaField\n mediaFallback\n plugins\n}\n\nfragment datasetFragment on Dataset {\n createdAt\n datasetId\n groupField\n id\n info\n lastLoadedAt\n mediaType\n name\n parentMediaType\n version\n appConfig {\n ...datasetAppConfigFragment\n }\n brainMethods {\n key\n version\n timestamp\n viewStages\n config {\n cls\n embeddingsField\n method\n patchesField\n supportsPrompts\n type\n maxK\n supportsLeastSimilarity\n }\n }\n defaultMaskTargets {\n target\n value\n }\n defaultSkeleton {\n labels\n edges\n }\n evaluations {\n key\n version\n timestamp\n viewStages\n config {\n cls\n predField\n gtField\n }\n }\n groupMediaTypes {\n name\n mediaType\n }\n maskTargets {\n name\n targets {\n target\n value\n }\n }\n skeletons {\n name\n labels\n edges\n }\n ...estimatedCountsFragment\n ...frameFieldsFragment\n ...groupSliceFragment\n ...indexesFragment\n ...mediaFieldsFragment\n ...mediaTypeFragment\n ...sampleFieldsFragment\n ...sidebarGroupsFragment\n ...viewFragment\n}\n\nfragment estimatedCountsFragment on Dataset {\n estimatedFrameCount\n estimatedSampleCount\n}\n\nfragment frameFieldsFragment on Dataset {\n frameFields {\n ftype\n subfield\n embeddedDocType\n path\n dbField\n description\n info\n }\n}\n\nfragment groupSliceFragment on Dataset {\n defaultGroupSlice\n}\n\nfragment indexesFragment on Dataset {\n frameIndexes {\n name\n unique\n key {\n field\n type\n }\n wildcardProjection {\n fields\n inclusion\n }\n }\n sampleIndexes {\n name\n unique\n key {\n field\n type\n }\n wildcardProjection {\n fields\n inclusion\n }\n }\n}\n\nfragment mediaFieldsFragment on Dataset {\n name\n appConfig {\n gridMediaField\n mediaFields\n modalMediaField\n mediaFallback\n }\n sampleFields {\n path\n }\n}\n\nfragment mediaTypeFragment on Dataset {\n mediaType\n}\n\nfragment sampleFieldsFragment on Dataset {\n sampleFields {\n ftype\n subfield\n embeddedDocType\n path\n dbField\n description\n info\n }\n}\n\nfragment savedViewsFragment on Query {\n savedViews(datasetName: $name) {\n id\n datasetId\n name\n slug\n description\n color\n viewStages\n createdAt\n lastModifiedAt\n lastLoadedAt\n }\n}\n\nfragment sidebarGroupsFragment on Dataset {\n datasetId\n appConfig {\n sidebarGroups {\n expanded\n paths\n name\n }\n }\n ...frameFieldsFragment\n ...sampleFieldsFragment\n}\n\nfragment stageDefinitionsFragment on Query {\n stageDefinitions {\n name\n params {\n name\n type\n default\n placeholder\n }\n }\n}\n\nfragment viewFragment on Dataset {\n stages(slug: $savedViewSlug, view: $view)\n viewCls\n viewName\n}\n\nfragment viewSchemaFragment on Query {\n schemaForViewStages(datasetName: $name, viewStages: $view) {\n fieldSchema {\n path\n ftype\n subfield\n embeddedDocType\n info\n description\n }\n frameFieldSchema {\n path\n ftype\n subfield\n embeddedDocType\n info\n description\n }\n }\n}\n" + "text": "query DatasetPageQuery(\n $count: Int\n $cursor: String\n $name: String!\n $extendedView: BSONArray!\n $savedViewSlug: String\n $search: String = \"\"\n $view: BSONArray!\n) {\n config {\n colorBy\n colorPool\n colorscale\n multicolorKeypoints\n showSkeletons\n }\n colorscale\n dataset(name: $name, view: $extendedView, savedViewSlug: $savedViewSlug) {\n name\n defaultGroupSlice\n appConfig {\n colorScheme {\n id\n colorBy\n colorPool\n multicolorKeypoints\n opacity\n showSkeletons\n defaultMaskTargetsColors {\n intTarget\n color\n }\n defaultColorscale {\n name\n list {\n value\n color\n }\n rgb\n }\n colorscales {\n path\n name\n list {\n value\n color\n }\n rgb\n }\n fields {\n colorByAttribute\n fieldColor\n path\n valueColors {\n color\n value\n }\n maskTargetsColors {\n intTarget\n color\n }\n }\n labelTags {\n fieldColor\n valueColors {\n color\n value\n }\n }\n }\n }\n ...datasetFragment\n id\n }\n ...NavFragment\n ...savedViewsFragment\n ...configFragment\n ...stageDefinitionsFragment\n ...viewSchemaFragment\n}\n\nfragment Analytics on Query {\n context\n dev\n doNotTrack\n uid\n version\n}\n\nfragment NavDatasets on Query {\n datasets(search: $search, first: $count, after: $cursor) {\n total\n edges {\n cursor\n node {\n name\n id\n __typename\n }\n }\n pageInfo {\n endCursor\n hasNextPage\n }\n }\n}\n\nfragment NavFragment on Query {\n ...Analytics\n ...NavDatasets\n}\n\nfragment colorSchemeFragment on ColorScheme {\n id\n colorBy\n colorPool\n multicolorKeypoints\n opacity\n showSkeletons\n labelTags {\n fieldColor\n valueColors {\n color\n value\n }\n }\n defaultMaskTargetsColors {\n intTarget\n color\n }\n defaultColorscale {\n name\n list {\n value\n color\n }\n rgb\n }\n colorscales {\n path\n name\n list {\n value\n color\n }\n rgb\n }\n fields {\n colorByAttribute\n fieldColor\n path\n valueColors {\n color\n value\n }\n maskTargetsColors {\n intTarget\n color\n }\n }\n}\n\nfragment configFragment on Query {\n config {\n colorBy\n colorPool\n colorscale\n disableFrameFiltering\n gridZoom\n enableQueryPerformance\n defaultQueryPerformance\n loopVideos\n mediaFallback\n multicolorKeypoints\n notebookHeight\n plugins\n showConfidence\n showIndex\n showLabel\n showSkeletons\n showTooltip\n theme\n timezone\n useFrameNumber\n }\n colorscale\n}\n\nfragment datasetAppConfigFragment on DatasetAppConfig {\n colorScheme {\n ...colorSchemeFragment\n id\n }\n defaultVisibilityLabels {\n include\n exclude\n }\n disableFrameFiltering\n dynamicGroupsTargetFrameRate\n gridMediaField\n mediaFields\n modalMediaField\n mediaFallback\n plugins\n}\n\nfragment datasetFragment on Dataset {\n createdAt\n datasetId\n groupField\n id\n info\n lastLoadedAt\n mediaType\n name\n parentMediaType\n version\n appConfig {\n ...datasetAppConfigFragment\n }\n brainMethods {\n key\n version\n timestamp\n viewStages\n config {\n cls\n embeddingsField\n method\n patchesField\n supportsPrompts\n type\n maxK\n supportsLeastSimilarity\n }\n }\n defaultMaskTargets {\n target\n value\n }\n defaultSkeleton {\n labels\n edges\n }\n evaluations {\n key\n version\n timestamp\n viewStages\n config {\n cls\n predField\n gtField\n }\n }\n groupMediaTypes {\n name\n mediaType\n }\n maskTargets {\n name\n targets {\n target\n value\n }\n }\n skeletons {\n name\n labels\n edges\n }\n ...estimatedCountsFragment\n ...frameFieldsFragment\n ...groupSliceFragment\n ...indexesFragment\n ...mediaFieldsFragment\n ...mediaTypeFragment\n ...sampleFieldsFragment\n ...sidebarGroupsFragment\n ...viewFragment\n}\n\nfragment estimatedCountsFragment on Dataset {\n estimatedFrameCount\n estimatedSampleCount\n}\n\nfragment frameFieldsFragment on Dataset {\n frameFields {\n ftype\n subfield\n embeddedDocType\n path\n dbField\n description\n info\n }\n}\n\nfragment groupSliceFragment on Dataset {\n defaultGroupSlice\n}\n\nfragment indexesFragment on Dataset {\n frameIndexes {\n name\n unique\n key {\n field\n type\n }\n wildcardProjection {\n fields\n inclusion\n }\n }\n sampleIndexes {\n name\n unique\n key {\n field\n type\n }\n wildcardProjection {\n fields\n inclusion\n }\n }\n}\n\nfragment mediaFieldsFragment on Dataset {\n name\n appConfig {\n gridMediaField\n mediaFields\n modalMediaField\n mediaFallback\n }\n sampleFields {\n path\n }\n}\n\nfragment mediaTypeFragment on Dataset {\n mediaType\n}\n\nfragment sampleFieldsFragment on Dataset {\n sampleFields {\n ftype\n subfield\n embeddedDocType\n path\n dbField\n description\n info\n }\n}\n\nfragment savedViewsFragment on Query {\n savedViews(datasetName: $name) {\n id\n datasetId\n name\n slug\n description\n color\n viewStages\n createdAt\n lastModifiedAt\n lastLoadedAt\n }\n}\n\nfragment sidebarGroupsFragment on Dataset {\n datasetId\n appConfig {\n sidebarGroups {\n expanded\n paths\n name\n }\n }\n ...frameFieldsFragment\n ...sampleFieldsFragment\n}\n\nfragment stageDefinitionsFragment on Query {\n stageDefinitions {\n name\n params {\n name\n type\n default\n placeholder\n }\n }\n}\n\nfragment viewFragment on Dataset {\n stages(slug: $savedViewSlug, view: $view)\n viewCls\n viewName\n}\n\nfragment viewSchemaFragment on Query {\n schemaForViewStages(datasetName: $name, viewStages: $view) {\n fieldSchema {\n path\n ftype\n subfield\n embeddedDocType\n info\n description\n }\n frameFieldSchema {\n path\n ftype\n subfield\n embeddedDocType\n info\n description\n }\n }\n}\n" } }; })(); diff --git a/app/packages/core/src/components/Dataset.tsx b/app/packages/core/src/components/Dataset.tsx index d533cbda6f..613ea31e96 100644 --- a/app/packages/core/src/components/Dataset.tsx +++ b/app/packages/core/src/components/Dataset.tsx @@ -1,3 +1,4 @@ +import { useTrackEvent } from "@fiftyone/analytics"; import { subscribe } from "@fiftyone/relay"; import { isModalActive } from "@fiftyone/state"; import React, { useEffect } from "react"; @@ -5,10 +6,9 @@ import { useRecoilValue } from "recoil"; import styled from "styled-components"; import ColorModal from "./ColorModal/ColorModal"; import { activeColorEntry } from "./ColorModal/state"; +import EventTracker from "./EventTracker"; import Modal from "./Modal"; import SamplesContainer from "./SamplesContainer"; -import EventTracker from "./EventTracker"; -import { useTrackEvent } from "@fiftyone/analytics"; const Container = styled.div` height: 100%; diff --git a/app/packages/core/src/components/Grid/Grid.tsx b/app/packages/core/src/components/Grid/Grid.tsx index 8948d8b1ff..e0200b6b06 100644 --- a/app/packages/core/src/components/Grid/Grid.tsx +++ b/app/packages/core/src/components/Grid/Grid.tsx @@ -10,9 +10,10 @@ import React, { useRef, useState, } from "react"; -import { useRecoilValue } from "recoil"; +import { useRecoilCallback, useRecoilValue } from "recoil"; import { v4 as uuid } from "uuid"; import { QP_WAIT, QueryPerformanceToastEvent } from "../QueryPerformanceToast"; +import { gridActivePathsLUT } from "../Sidebar/useDetectNewActiveLabelFields"; import { gridCrop, gridSpacing, pageParameters } from "./recoil"; import useAt from "./useAt"; import useEscape from "./useEscape"; @@ -46,6 +47,16 @@ function Grid() { const setSample = fos.useExpandSample(store); const getFontSize = useFontSize(id); + const getCurrentActiveLabelFields = useRecoilCallback( + ({ snapshot }) => + () => { + return snapshot + .getLoadable(fos.activeLabelFields({ modal: false })) + .getValue(); + }, + [] + ); + const spotlight = useMemo(() => { /** SPOTLIGHT REFRESHER */ reset; @@ -61,6 +72,7 @@ function Grid() { const looker = lookerStore.get(id.description); looker?.destroy(); lookerStore.delete(id.description); + gridActivePathsLUT.delete(id.description); }, detach: (id) => { const looker = lookerStore.get(id.description); @@ -101,6 +113,18 @@ function Grid() { ); lookerStore.set(id.description, looker); looker.attach(element, dimensions); + + // initialize active paths tracker + const currentActiveLabelFields = getCurrentActiveLabelFields(); + if ( + currentActiveLabelFields && + !gridActivePathsLUT.has(id.description) + ) { + gridActivePathsLUT.set( + id.description, + new Set(currentActiveLabelFields) + ); + } }, scrollbar: true, spacing, diff --git a/app/packages/core/src/components/Grid/useRefreshers.ts b/app/packages/core/src/components/Grid/useRefreshers.ts index 1fa1b38d09..0f64d62454 100644 --- a/app/packages/core/src/components/Grid/useRefreshers.ts +++ b/app/packages/core/src/components/Grid/useRefreshers.ts @@ -4,6 +4,7 @@ import { LRUCache } from "lru-cache"; import { useEffect, useMemo } from "react"; import uuid from "react-uuid"; import { useRecoilValue } from "recoil"; +import { gridActivePathsLUT } from "../Sidebar/useDetectNewActiveLabelFields"; import { gridAt, gridOffset, gridPage } from "./recoil"; const MAX_LRU_CACHE_ITEMS = 510; @@ -64,18 +65,20 @@ export default function useRefreshers() { return uuid(); }, [layoutReset, pageReset]); - useEffect( - () => - subscribe(({ event }, { reset }) => { - if (event === "fieldVisibility") return; + useEffect(() => { + const unsubscribe = subscribe(({ event }, { reset }) => { + if (event === "fieldVisibility") return; - // if not a modal page change, reset the grid location - reset(gridAt); - reset(gridPage); - reset(gridOffset); - }), - [] - ); + // if not a modal page change, reset the grid location + reset(gridAt); + reset(gridPage); + reset(gridOffset); + }); + + return () => { + unsubscribe(); + }; + }, []); const lookerStore = useMemo(() => { /** LOOKER STORE REFRESHER */ @@ -83,8 +86,9 @@ export default function useRefreshers() { /** LOOKER STORE REFRESHER */ return new LRUCache({ - dispose: (looker) => { + dispose: (looker, id) => { looker.destroy(); + gridActivePathsLUT.delete(id); }, max: MAX_LRU_CACHE_ITEMS, noDisposeOnSet: true, diff --git a/app/packages/core/src/components/Grid/useSelect.ts b/app/packages/core/src/components/Grid/useSelect.ts index 7958c8b219..2ec22dc1b5 100644 --- a/app/packages/core/src/components/Grid/useSelect.ts +++ b/app/packages/core/src/components/Grid/useSelect.ts @@ -1,8 +1,11 @@ +import { VideoLooker } from "@fiftyone/looker"; import type Spotlight from "@fiftyone/spotlight"; import * as fos from "@fiftyone/state"; import type { LRUCache } from "lru-cache"; import { useEffect } from "react"; import { useRecoilValue } from "recoil"; +import { useDetectNewActiveLabelFields } from "../Sidebar/useDetectNewActiveLabelFields"; +import { RENDER_STATUS_PENDING } from "@fiftyone/looker/src/worker/shared"; export default function useSelect( getFontSize: () => number, @@ -12,6 +15,10 @@ export default function useSelect( ) { const { init, deferred } = fos.useDeferrer(); + const { getNewFields } = useDetectNewActiveLabelFields({ + modal: false, + }); + const selected = useRecoilValue(fos.selectedSamples); useEffect(() => { deferred(() => { @@ -24,11 +31,62 @@ export default function useSelect( } retained.add(id.description); + + const newFieldsIfAny = getNewFields(id.description); + + const overlays = + instance instanceof VideoLooker + ? instance.pluckedOverlays + : instance.sampleOverlays; + instance.updateOptions({ ...options, fontSize, selected: selected.has(id.description), }); + + // rerender looker if active fields have changed and have never been rendered before + if (newFieldsIfAny) { + const thisInstanceOverlays = overlays?.filter( + (o) => + o.field && + (o.label?.mask_path?.length > 0 || + o.label?.map_path?.length > 0 || + o.label?.mask || + o.label?.map) && + newFieldsIfAny.includes(o.field) + ); + + thisInstanceOverlays?.forEach((o) => { + if (o.label) { + // "pending" means we're marking this label for rendering / painting + // even if it's interrupted, say by unchecking sidebar + o.label.renderStatus = RENDER_STATUS_PENDING; + } + }); + + // important we "reconcile" here so that "pending" status percolates to + // draw function of labels + instance.updateOptions({ ...options }); + + if (thisInstanceOverlays?.length > 0) { + instance.refreshSample(newFieldsIfAny); + } + } else { + // if there're any labels marked "pending", render them + const pending = overlays?.filter( + (o) => o.field && o.label?.renderStatus === RENDER_STATUS_PENDING + ); + + if (pending?.length > 0) { + const rerenderFields = pending.map((o) => o.field!); + // `refreshSample` calls `updateOptions` internally + instance.refreshSample(rerenderFields); + } + instance.updateOptions({ + ...options, + }); + } }); for (const id of store.keys()) { diff --git a/app/packages/core/src/components/Modal/ImaVidLooker.tsx b/app/packages/core/src/components/Modal/ImaVidLooker.tsx index a2134b2702..26447c304c 100644 --- a/app/packages/core/src/components/Modal/ImaVidLooker.tsx +++ b/app/packages/core/src/components/Modal/ImaVidLooker.tsx @@ -1,6 +1,5 @@ import { useTheme } from "@fiftyone/components"; -import { AbstractLooker, ImaVidLooker } from "@fiftyone/looker"; -import { BaseState } from "@fiftyone/looker/src/state"; +import { ImaVidLooker } from "@fiftyone/looker"; import { FoTimelineConfig, useCreateTimeline } from "@fiftyone/playback"; import { useDefaultTimelineNameImperative } from "@fiftyone/playback/src/lib/use-default-timeline-name"; import { Timeline } from "@fiftyone/playback/src/views/Timeline"; @@ -24,6 +23,7 @@ import { useModalContext, } from "./hooks"; import useKeyEvents from "./use-key-events"; +import { useImavidModalSelectiveRendering } from "./use-modal-selective-rendering"; import { shortcutToHelpItems } from "./utils"; interface ImaVidLookerReactProps { @@ -58,12 +58,12 @@ export const ImaVidLookerReact = React.memo( const { activeLookerRef, setActiveLookerRef } = useModalContext(); const imaVidLookerRef = - activeLookerRef as unknown as React.MutableRefObject; + activeLookerRef as React.MutableRefObject; const looker = React.useMemo( () => createLooker.current(sampleDataWithExtraParams), [reset, createLooker, selectedMediaField] - ) as AbstractLooker; + ) as ImaVidLooker; useEffect(() => { setModalLooker(looker); @@ -74,7 +74,7 @@ export const ImaVidLookerReact = React.memo( useEffect(() => { if (looker) { - setActiveLookerRef(looker as fos.Lookers); + setActiveLookerRef(looker); } }, [looker]); @@ -317,6 +317,8 @@ export const ImaVidLookerReact = React.memo( return () => clearInterval(intervalId); }, [looker]); + useImavidModalSelectiveRendering(id, looker); + return (
{ return useRecoilCallback(({ set }) => async (event: CustomEvent) => { @@ -27,8 +28,9 @@ interface LookerProps { } const ModalLookerNoTimeline = React.memo((props: LookerProps) => { - const { id, ref } = useLooker(props); + const { id, ref, looker } = useLooker(props); const theme = useTheme(); + useImageModalSelectiveRendering(id, looker); return (
{ return sample.frameRate; }, [sample]); + useVideoModalSelectiveRendering(id, looker); + useEffect(() => { const load = () => { const duration = looker.getVideo().duration; diff --git a/app/packages/core/src/components/Modal/use-modal-selective-rendering.ts b/app/packages/core/src/components/Modal/use-modal-selective-rendering.ts new file mode 100644 index 0000000000..7f151919fd --- /dev/null +++ b/app/packages/core/src/components/Modal/use-modal-selective-rendering.ts @@ -0,0 +1,99 @@ +import { ImaVidLooker, VideoLooker } from "@fiftyone/looker"; +import { Lookers, useLookerOptions } from "@fiftyone/state"; +import { useCallback, useEffect, useRef } from "react"; +import { useDetectNewActiveLabelFields } from "../Sidebar/useDetectNewActiveLabelFields"; + +export const useImageModalSelectiveRendering = ( + id: string, + looker: Lookers +) => { + const lookerOptions = useLookerOptions(true); + + const { getNewFields } = useDetectNewActiveLabelFields({ + modal: true, + }); + + useEffect(() => { + if (!looker) { + return; + } + + const newFieldsIfAny = getNewFields(id); + + if (newFieldsIfAny) { + looker?.refreshSample(newFieldsIfAny); + } + }, [id, lookerOptions.activePaths, looker, getNewFields]); +}; + +export const useImavidModalSelectiveRendering = ( + id: string, + looker: ImaVidLooker +) => { + const { getNewFields } = useDetectNewActiveLabelFields({ + modal: true, + }); + + const lookerOptions = useLookerOptions(true); + + // this is for default view + // subscription below will not have triggered for the first frame + useImageModalSelectiveRendering(id, looker); + + // using weak heuristic to detect coloring changes + // this is not perfect, but should be good enough + const getColoringHash = useCallback(() => { + return lookerOptions?.coloring?.targets.join("-") ?? ""; + }, [lookerOptions]); + + useEffect(() => { + const unsub = looker.subscribeToState( + "currentFrameNumber", + (currentFrameNumber: number) => { + if (!looker.thisFrameSample?.sample) { + return; + } + + const thisFrameId = `${id}-${currentFrameNumber}-${getColoringHash()}`; + + const newFieldsIfAny = getNewFields(thisFrameId); + + if (newFieldsIfAny) { + looker.refreshSample(newFieldsIfAny, currentFrameNumber); + } else { + // repainting labels should be sufficient + looker.refreshOverlaysToCurrentFrame(); + } + } + ); + + return () => { + unsub(); + }; + }, [getNewFields, getColoringHash, looker]); +}; + +export const useVideoModalSelectiveRendering = ( + id: string, + looker: VideoLooker +) => { + const { getNewFields } = useDetectNewActiveLabelFields({ + modal: true, + }); + + const lookerOptions = useLookerOptions(true); + + useEffect(() => { + if (!looker) { + return; + } + + const newFieldsIfAny = getNewFields(id); + + if (newFieldsIfAny) { + // todo: no granular refreshing for video looker + // it'd require selective re-processing of frames in the buffer + looker?.refreshSample(); + } + }, [id, lookerOptions.activePaths, looker, getNewFields]); +}; diff --git a/app/packages/core/src/components/Sidebar/syncAndGetNewLabels.test.ts b/app/packages/core/src/components/Sidebar/syncAndGetNewLabels.test.ts new file mode 100644 index 0000000000..371b81db4b --- /dev/null +++ b/app/packages/core/src/components/Sidebar/syncAndGetNewLabels.test.ts @@ -0,0 +1,87 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { syncAndGetNewLabels } from "./syncAndGetNewLabels"; + +describe("syncAndGetNewFields", () => { + let lut: Map>; + + beforeEach(() => { + lut = new Map(); + }); + + it("returns null if currentActiveLabelFields is empty", () => { + const id = "looker1"; + const currentActiveLabelFields = new Set(); + + const result = syncAndGetNewLabels(id, lut, currentActiveLabelFields); + expect(result).toBeNull(); + // lut shouldn't be modified + expect(lut.has(id)).toBe(false); + }); + + it("returns newly added fields if lut is empty", () => { + const id = "looker1"; + const currentActiveLabelFields = new Set(["segmentation1"]); + + const result = syncAndGetNewLabels(id, lut, currentActiveLabelFields); + expect(result).toEqual(["segmentation1"]); + + // lut should now have the newly added fields + const storedFields = lut.get(id); + expect(storedFields).toEqual(new Set(["segmentation1"])); + }); + + it("returns newly added fields for a new looker (lut doesn't have the looker yet)", () => { + lut.set("existingLooker", new Set(["seg1"])); + + const currentActiveLabelFields = new Set(["seg1", "heatmap1"]); + const newLookerId = "newLooker"; + + const result = syncAndGetNewLabels( + newLookerId, + lut, + currentActiveLabelFields + ); + expect(result).toEqual(["seg1", "heatmap1"]); + + // Check that they're actually stored in the LUT now + expect(lut.get(newLookerId)).toEqual(new Set(["seg1", "heatmap1"])); + expect(lut.get("existingLooker")).toEqual(new Set(["seg1"])); + }); + + it("returns null if lut already has the same fields (no new fields)", () => { + const id = "looker2"; + lut.set(id, new Set(["seg1", "seg2"])); + + const currentActiveLabelFields = new Set(["seg1", "seg2"]); + const result = syncAndGetNewLabels(id, lut, currentActiveLabelFields); + + expect(result).toBeNull(); + // lut remains the same, no changes + expect(lut.get(id)).toEqual(new Set(["seg1", "seg2"])); + }); + + it("returns newly added fields if some fields are missing in lut", () => { + const id = "looker2"; + lut.set(id, new Set(["field1"])); + + const currentActiveLabelFields = new Set(["field1", "field2"]); + const result = syncAndGetNewLabels(id, lut, currentActiveLabelFields); + + expect(result).toEqual(["field2"]); // only new field + expect(lut.get(id)).toEqual(new Set(["field1", "field2"])); + }); + + it("doesn't keep returning newly added fields after lut is updated", () => { + const id = "looker3"; + lut.set(id, new Set(["field1"])); + + // First call adds "field2" + let result = syncAndGetNewLabels(id, lut, new Set(["field1", "field2"])); + expect(result).toEqual(["field2"]); + expect(lut.get(id)).toEqual(new Set(["field1", "field2"])); + + // Second call with the same fields should return null + result = syncAndGetNewLabels(id, lut, new Set(["field1", "field2"])); + expect(result).toBeNull(); + }); +}); diff --git a/app/packages/core/src/components/Sidebar/syncAndGetNewLabels.ts b/app/packages/core/src/components/Sidebar/syncAndGetNewLabels.ts new file mode 100644 index 0000000000..89d7cf7ba7 --- /dev/null +++ b/app/packages/core/src/components/Sidebar/syncAndGetNewLabels.ts @@ -0,0 +1,48 @@ +import { CachedLabels, LookerId } from "./useDetectNewActiveLabelFields"; + +/** + * Synchronizes which label fields a given looker has seen so far. Returns any + * new (not-yet-cached) fields as an array, or null if there are no new fields. + * + * @param lookerId - unique Looker ID + * @param lut - look-up table tracking which fields each looker has already rendered + * @param currentActiveLabelFields - set of active label fields for the modal/looker + * @returns array of newly added label fields, or null if there are none + */ +export const syncAndGetNewLabels = ( + lookerId: string, + lut: Map, + currentActiveLabelFields: Set +): string[] | null => { + if (currentActiveLabelFields.size === 0) { + return null; + } + + const cachedFieldsForLooker = lut.get(lookerId); + let newFields: string[] = []; + + if (cachedFieldsForLooker) { + // Collect only the fields that aren't in the cache + for (const field of currentActiveLabelFields) { + if (!cachedFieldsForLooker.has(field)) { + newFields.push(field); + } + } + } else { + // If there is no cache for this looker, everything is new + newFields = Array.from(currentActiveLabelFields); + } + + if (newFields.length === 0) { + // No additions, so no update needed + return null; + } + + // Update the cache by merging new fields into the existing set (or create a new set) + lut.set( + lookerId, + new Set([...(cachedFieldsForLooker ?? []), ...currentActiveLabelFields]) + ); + + return newFields; +}; diff --git a/app/packages/core/src/components/Sidebar/useDetectNewActiveLabelFields.ts b/app/packages/core/src/components/Sidebar/useDetectNewActiveLabelFields.ts new file mode 100644 index 0000000000..9025db7bea --- /dev/null +++ b/app/packages/core/src/components/Sidebar/useDetectNewActiveLabelFields.ts @@ -0,0 +1,74 @@ +import { activeLabelFields, datasetName } from "@fiftyone/state"; +import { useCallback, useEffect } from "react"; +import { useRecoilValue } from "recoil"; +import { syncAndGetNewLabels } from "./syncAndGetNewLabels"; + +export type LookerId = string; +export type CachedLabels = Set; + +export const gridActivePathsLUT = new Map(); +export const modalActivePathsLUT = new Map(); + +/** + * Detects newly introduced active label fields for a given looker. Returns a + * callback that, given a looker ID, merges and returns any fields not yet in + * the cache. Clears its cache when unmounted. + * + * @param modal - Whether this hook is used in a modal context + * @returns A function that accepts a looker ID and returns newly added fields + * or null if there are none + */ +export const useDetectNewActiveLabelFields = ({ + modal, +}: { + modal: boolean; +}) => { + const activeLabelFieldsValue = useRecoilValue(activeLabelFields({ modal })); + + const datasetNameValue = useRecoilValue(datasetName); + + // reset when dataset changes + useEffect(() => { + if (modal) { + modalActivePathsLUT.clear(); + } else { + gridActivePathsLUT.clear(); + } + }, [datasetNameValue]); + + const getNewFields = useCallback( + (id: string) => { + return syncAndGetNewLabels( + id, + modal ? modalActivePathsLUT : gridActivePathsLUT, + new Set(activeLabelFieldsValue) + ); + }, + [activeLabelFieldsValue] + ); + + const getExistingFields = useCallback( + (id: string) => { + const mayBeFields = modal + ? modalActivePathsLUT.get(id) + : gridActivePathsLUT.get(id); + return mayBeFields ? Array.from(mayBeFields) : []; + }, + [modal] + ); + + /** + * clear look up table when component unmounts + */ + useEffect(() => { + return () => { + if (modal) { + modalActivePathsLUT.clear(); + } else { + gridActivePathsLUT.clear(); + } + }; + }, [modal]); + + return { getNewFields, getExistingFields }; +}; diff --git a/app/packages/looker/src/elements/common/error.ts b/app/packages/looker/src/elements/common/error.ts index b2bc534ab5..3de014569a 100644 --- a/app/packages/looker/src/elements/common/error.ts +++ b/app/packages/looker/src/elements/common/error.ts @@ -76,7 +76,9 @@ export class ErrorElement extends BaseElement { } else { const text = document.createElement("p"); const textDiv = document.createElement("div"); - text.innerText = "Something went wrong"; + text.innerText = + "Something went wrong" + + (error["message"] ? `: ${error["message"]}` : "."); textDiv.appendChild(text); this.errorElement.appendChild(textDiv); } diff --git a/app/packages/looker/src/lookers/abstract.ts b/app/packages/looker/src/lookers/abstract.ts index 820a0fe3d8..714596ff89 100644 --- a/app/packages/looker/src/lookers/abstract.ts +++ b/app/packages/looker/src/lookers/abstract.ts @@ -1,3 +1,4 @@ +import { Lookers } from "@fiftyone/state"; import { AppError, DATE_FIELD, @@ -11,7 +12,6 @@ import { withPath, } from "@fiftyone/utilities"; import { isEmpty } from "lodash"; -import { v4 as uuid } from "uuid"; import { BASE_ALPHA, DASH_LENGTH, @@ -42,9 +42,13 @@ import { mergeUpdates, snapBox, } from "../util"; +import { v4 as uuid } from "uuid"; import { ProcessSample } from "../worker"; +import { AsyncLabelsRenderingManager } from "../worker/async-labels-rendering-manager"; import { LookerUtils } from "./shared"; -import { retrieveArrayBuffers } from "./utils"; +import { retrieveTransferables } from "./utils"; + +const UPDATING_SAMPLES_IDS = new Set(); const LABEL_LISTS_PATH = new Set(withPath(LABELS_PATH, LABEL_LISTS)); const LABEL_LIST_KEY = Object.fromEntries( @@ -93,17 +97,24 @@ export abstract class AbstractLooker< private readonly rootEvents: Events; protected readonly abortController: AbortController; - protected sampleOverlays: Overlay[]; protected currentOverlays: Overlay[]; - protected pluckedOverlays: Overlay[]; protected sample: S; - protected state: State; protected readonly updater: StateUpdate; private batchMergedUpdates: Partial = {}; private isBatching = false; private isCommittingBatchUpdates = false; + public uuid = uuid(); + + /** @internal */ + state: State; + + sampleOverlays: Overlay[]; + pluckedOverlays: Overlay[]; + + protected asyncLabelsRenderingManager: AsyncLabelsRenderingManager; + constructor( sample: S, config: State["config"], @@ -156,6 +167,10 @@ export abstract class AbstractLooker< 3500 ); + this.asyncLabelsRenderingManager = new AsyncLabelsRenderingManager( + this as unknown as Lookers + ); + this.init(); } @@ -517,7 +532,71 @@ export abstract class AbstractLooker< abstract updateOptions(options: Partial): void; updateSample(sample: Sample) { - this.loadSample(sample, retrieveArrayBuffers(this.sampleOverlays)); + // todo: sometimes instance in spotlight?.updateItems() is defined but has no ref to sample + // this crashes the app. this is a bug and should be fixed + if (!this.sample) { + return; + } + + const id = sample.id ?? sample._id; + const updateTimeoutMs = 10000; + + if (UPDATING_SAMPLES_IDS.has(id)) { + UPDATING_SAMPLES_IDS.delete(id); + this.cleanOverlays(true); + + // to prevent deadlock, we'll remove the id from the set after a timeout + const timeoutId = setTimeout(() => { + UPDATING_SAMPLES_IDS.delete(id); + }, updateTimeoutMs); + + queueMicrotask(() => { + try { + this.updateSample(sample); + clearTimeout(timeoutId); + } catch (e) { + UPDATING_SAMPLES_IDS.delete(id); + this.updater({ error: e }); + } + }); + return; + } + + UPDATING_SAMPLES_IDS.add(id); + + this.loadSample(sample, retrieveTransferables(this.sampleOverlays)); + } + + refreshSample(renderLabels: string[] | null = null) { + // todo: sometimes instance in spotlight?.updateItems() is defined but has no ref to sample + // this crashes the app. this is a bug and should be fixed + if (!this.sample) { + return; + } + + if (!renderLabels?.length) { + this.updateSample(this.sample); + return; + } + + this.asyncLabelsRenderingManager + .enqueueLabelPaintingJob({ + sample: this.sample, + labels: renderLabels, + }) + .then(({ sample, coloring }) => { + this.sample = sample; + this.state.options.coloring = coloring; + this.loadOverlays(sample); + + // to run looker reconciliation + this.updater({ + overlaysPrepared: true, + }); + }) + .catch((error) => { + this.updater({ error }); + }); } getSample(): Promise { @@ -701,9 +780,9 @@ export abstract class AbstractLooker< ); } - protected cleanOverlays() { + protected cleanOverlays(setTargetsToNull = false) { for (const overlay of this.sampleOverlays ?? []) { - overlay.cleanup?.(); + overlay.cleanup?.(setTargetsToNull); } } @@ -726,6 +805,8 @@ export abstract class AbstractLooker< reloading: false, }); labelsWorker.removeEventListener("message", listener); + + UPDATING_SAMPLES_IDS.delete(sample.id ?? sample._id); } }; @@ -742,6 +823,7 @@ export abstract class AbstractLooker< sources: this.state.config.sources, schema: this.state.config.fieldSchema, uuid: messageUUID, + activePaths: this.state.options.activePaths, } as ProcessSample; try { diff --git a/app/packages/looker/src/lookers/frame-reader.ts b/app/packages/looker/src/lookers/frame-reader.ts index ce0489dc43..db7e81fe3a 100644 --- a/app/packages/looker/src/lookers/frame-reader.ts +++ b/app/packages/looker/src/lookers/frame-reader.ts @@ -32,6 +32,7 @@ export interface Frame { } interface AcquireReaderOptions { + activePaths: string[]; addFrame: (frameNumber: number, frame: Frame) => void; addFrameBuffers: (range: [number, number]) => void; coloring: Coloring; @@ -88,6 +89,7 @@ export const { acquireReader, clearReader } = (() => { view, group, schema, + activePaths, }: AcquireReaderOptions): string => { nextRange = [frameNumber, Math.min(frameCount, CHUNK_SIZE + frameNumber)]; const subscription = uuid(); @@ -130,6 +132,7 @@ export const { acquireReader, clearReader } = (() => { requestingFrames = true; frameReader.postMessage({ method: "setStream", + activePaths, sampleId, frameCount, frameNumber, diff --git a/app/packages/looker/src/lookers/imavid/index.ts b/app/packages/looker/src/lookers/imavid/index.ts index dc75a3b4f2..914dd98eba 100644 --- a/app/packages/looker/src/lookers/imavid/index.ts +++ b/app/packages/looker/src/lookers/imavid/index.ts @@ -1,3 +1,5 @@ +import { syncAndGetNewLabels } from "@fiftyone/core/src/components/Sidebar/syncAndGetNewLabels"; +import { gridActivePathsLUT } from "@fiftyone/core/src/components/Sidebar/useDetectNewActiveLabelFields"; import { BufferManager } from "@fiftyone/utilities"; import { getImaVidElements } from "../../elements"; import { IMAVID_SHORTCUTS } from "../../elements/common/actions"; @@ -30,11 +32,42 @@ export class ImaVidLooker extends AbstractLooker { private unsubscribe: ReturnType; init() { + // we have other mechanism for the modal + if (!this.state.config.thumbnail) { + return; + } + // subscribe to frame number and update sample when frame number changes - this.unsubscribe = this.subscribeToState("currentFrameNumber", () => { - this.thisFrameSample?.sample && - this.updateSample(this.thisFrameSample.sample); - }); + this.unsubscribe = this.subscribeToState( + "currentFrameNumber", + (currentFrameNumber: number) => { + const thisFrameId = `${ + this.uuid + }-${currentFrameNumber}-${this.state.options.coloring.targets.join( + "-" + )}`; + + if ( + gridActivePathsLUT.has(thisFrameId) && + this.thisFrameSample?.sample + ) { + this.refreshOverlaysToCurrentFrame(); + } else { + const newFieldsIfAny = syncAndGetNewLabels( + thisFrameId, + gridActivePathsLUT, + new Set(this.options.activePaths) + ); + + if (newFieldsIfAny && currentFrameNumber > 0) { + this.refreshSample(newFieldsIfAny, currentFrameNumber); + } else { + // worst case, only here for fail-safe + this.refreshSample(); + } + } + } + ); } get thisFrameSample() { @@ -195,9 +228,75 @@ export class ImaVidLooker extends AbstractLooker { if (reload) { this.updater({ options, reloading: this.state.disabled }); - this.updateSample(this.sample); + if (this.config.thumbnail) { + // `useImavidModalSelectiveRendering` takes care of it for modal + this.updateSample(this.sample); + } } else { this.updater({ options, disabled: false }); } } + + refreshOverlaysToCurrentFrame() { + let { image: _cachedImage, ...thisFrameSample } = + this.frameStoreController.store.getSampleAtFrame( + this.frameNumber + )?.sample; + + if (!thisFrameSample) { + thisFrameSample = this.sample; + } + + this.loadOverlays(thisFrameSample); + } + + refreshSample(renderLabels: string[] | null = null, frameNumber?: number) { + // todo: sometimes instance in spotlight?.updateItems() is defined but has no ref to sample + // this crashes the app. this is a bug and should be fixed + if (!this.sample) { + return; + } + + if (!renderLabels?.length) { + this.updateSample(this.sample); + return; + } + + const sampleIdFromFramesStore = + this.frameStoreController.store.frameIndex.get(frameNumber); + + let sample: Sample; + + // if sampleIdFromFramesStore is not found, it means we're in grid thumbnail view + if (sampleIdFromFramesStore) { + const { image: _cachedImage, ...sampleWithoutImage } = + this.frameStoreController.store.samples.get(sampleIdFromFramesStore); + sample = sampleWithoutImage.sample; + } else { + sample = this.sample; + } + + this.asyncLabelsRenderingManager + .enqueueLabelPaintingJob({ + sample: sample as Sample, + labels: renderLabels, + }) + .then(({ sample, coloring }) => { + if (sampleIdFromFramesStore) { + this.frameStoreController.store.updateSample( + sampleIdFromFramesStore, + sample + ); + } else { + this.sample = sample; + } + this.state.options.coloring = coloring; + this.loadOverlays(sample); + + // to run looker reconciliation + this.updater({ + overlaysPrepared: true, + }); + }); + } } diff --git a/app/packages/looker/src/lookers/utils.test.ts b/app/packages/looker/src/lookers/utils.test.ts index ed2b9a1100..b3bfc74967 100644 --- a/app/packages/looker/src/lookers/utils.test.ts +++ b/app/packages/looker/src/lookers/utils.test.ts @@ -7,7 +7,7 @@ import KeypointOverlay from "../overlays/keypoint"; import PolylineOverlay from "../overlays/polyline"; import SegmentationOverlay from "../overlays/segmentation"; import type { Buffers } from "../state"; -import { hasFrame, retrieveArrayBuffers } from "./utils"; +import { hasFrame, retrieveTransferables } from "./utils"; describe("looker utilities", () => { it("determines frame availability given a buffer list", () => { @@ -26,25 +26,25 @@ describe("looker utilities", () => { it("retrieves array buffers without errors", () => { expect( - retrieveArrayBuffers([new ClassificationsOverlay([])]) + retrieveTransferables([new ClassificationsOverlay([])]) ).toStrictEqual([]); expect( - retrieveArrayBuffers([new DetectionOverlay("ground_truth", {})]) + retrieveTransferables([new DetectionOverlay("ground_truth", {})]) ).toStrictEqual([]); expect( - retrieveArrayBuffers([ + retrieveTransferables([ new HeatmapOverlay("ground_truth", { id: "", tags: [] }), ]) ).toStrictEqual([]); expect( - retrieveArrayBuffers([new KeypointOverlay("ground_truth", {})]) + retrieveTransferables([new KeypointOverlay("ground_truth", {})]) ).toStrictEqual([]); expect( - retrieveArrayBuffers([ + retrieveTransferables([ new PolylineOverlay("ground_truth", { id: "", closed: false, @@ -56,13 +56,13 @@ describe("looker utilities", () => { ).toStrictEqual([]); expect( - retrieveArrayBuffers([ + retrieveTransferables([ new SegmentationOverlay("ground_truth", { id: "", tags: [] }), ]) ).toStrictEqual([]); expect( - retrieveArrayBuffers([new TemporalDetectionOverlay([])]) + retrieveTransferables([new TemporalDetectionOverlay([])]) ).toStrictEqual([]); }); }); diff --git a/app/packages/looker/src/lookers/utils.ts b/app/packages/looker/src/lookers/utils.ts index 4d56ee520f..5f37b1a708 100644 --- a/app/packages/looker/src/lookers/utils.ts +++ b/app/packages/looker/src/lookers/utils.ts @@ -10,12 +10,12 @@ export const hasFrame = (buffers: Buffers, frameNumber: number) => { ); }; -export const retrieveArrayBuffers = ( +export const retrieveTransferables = ( overlays?: Overlay[] ) => { // collect any mask targets array buffer that overlays might have // we'll transfer that to the worker instead of copying it - const arrayBuffers: ArrayBuffer[] = []; + const transferables: Transferable[] = []; for (const overlay of overlays ?? []) { let overlayData: LabelMask = null; @@ -30,28 +30,31 @@ export const retrieveArrayBuffers = ( } const buffer = overlayData?.data?.buffer; - - if (!buffer) { - continue; - } - - // check for detached buffer (happens if user is switching colors too fast) - // note: ArrayBuffer.prototype.detached is a new browser API - if (typeof buffer.detached !== "undefined") { - if (buffer.detached) { - // most likely sample is already being processed, skip update - return []; + const bitmap = overlayData?.bitmap; + + if (buffer) { + // check for detached buffer (happens if user is switching colors too fast) + // note: ArrayBuffer.prototype.detached is a new browser API + if (typeof buffer.detached !== "undefined") { + if (buffer.detached) { + // most likely sample is already being processed, skip update + return []; + } + + transferables.push(buffer); + } else if (buffer.byteLength) { + // hope we don't run into this edge case (old browser) + // sometimes detached buffers have bytelength > 0 + // if we run into this case, we'll just attempt to transfer the buffer + // might get a DataCloneError if user is switching colors too fast + transferables.push(buffer); } + } - arrayBuffers.push(buffer); - } else if (buffer.byteLength) { - // hope we don't run into this edge case (old browser) - // sometimes detached buffers have bytelength > 0 - // if we run into this case, we'll just attempt to transfer the buffer - // might get a DataCloneError if user is switching colors too fast - arrayBuffers.push(buffer); + if (bitmap?.width || bitmap?.height) { + transferables.push(bitmap); } } - return arrayBuffers; + return transferables; }; diff --git a/app/packages/looker/src/lookers/video.ts b/app/packages/looker/src/lookers/video.ts index d1f94328b4..51172f9d82 100644 --- a/app/packages/looker/src/lookers/video.ts +++ b/app/packages/looker/src/lookers/video.ts @@ -257,6 +257,7 @@ export class VideoLooker extends AbstractLooker { addFrameBuffers: (range) => { this.state.buffers = addToBuffers(range, this.state.buffers); }, + activePaths: this.state.options.activePaths, coloring: this.state.options.coloring, customizeColorSetting: this.state.options.customizeColorSetting, dispatchEvent: (event, detail) => this.dispatchEvent(event, detail), diff --git a/app/packages/looker/src/overlays/base.ts b/app/packages/looker/src/overlays/base.ts index 94a7eb6683..e7530c7810 100644 --- a/app/packages/looker/src/overlays/base.ts +++ b/app/packages/looker/src/overlays/base.ts @@ -5,6 +5,7 @@ import { getCls, sizeBytesEstimate } from "@fiftyone/utilities"; import { OverlayMask } from "../numpy"; import type { BaseState, Coordinates, NONFINITE } from "../state"; +import { DenseLabelRenderStatus } from "../worker/shared"; import { getLabelColor, shouldShowLabelTag } from "./util"; // in numerical order (CONTAINS_BORDER takes precedence over CONTAINS_CONTENT) @@ -19,6 +20,7 @@ export interface BaseLabel { frame_number?: number; tags: string[]; index?: number; + renderStatus?: DenseLabelRenderStatus; } export interface PointInfo