From 209a2781e91e77331e9748c2cd70c58d05cbcaf5 Mon Sep 17 00:00:00 2001 From: Ihor Dykhta Date: Mon, 27 Jan 2025 13:37:47 +0200 Subject: [PATCH] [feat] emulate geoarrow.wkb and geoarrow.linestring support in trip-layer Signed-off-by: Ihor Dykhta --- src/layers/src/geojson-layer/geojson-layer.ts | 39 ++- src/layers/src/geojson-layer/geojson-utils.ts | 6 +- src/layers/src/trip-layer/trip-layer.ts | 313 +++++++++++++----- src/layers/src/trip-layer/trip-utils.ts | 90 +++++ 4 files changed, 347 insertions(+), 101 deletions(-) diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts index 656bf9f92f..c19b7c1537 100644 --- a/src/layers/src/geojson-layer/geojson-layer.ts +++ b/src/layers/src/geojson-layer/geojson-layer.ts @@ -53,7 +53,7 @@ import { ProtoDatasetField, LayerColumn } from '@kepler.gl/types'; -import {KeplerTable} from '@kepler.gl/table'; +import {KeplerTable, Datasets} from '@kepler.gl/table'; import {DataContainerInterface, ArrowDataContainer} from '@kepler.gl/utils'; import {FilterArrowExtension} from '@kepler.gl/deckgl-layers'; import GeojsonInfoModalFactory from './geojson-info-modal'; @@ -182,7 +182,7 @@ type ObjectInfo = { export const featureAccessor = ({geojson}: GeoJsonLayerColumnsConfig) => (dc: DataContainerInterface) => - d => + (d: {index: number}) => dc.valueAt(d.index, geojson.fieldIdx); const geoColumnAccessor = @@ -232,15 +232,17 @@ const SUPPORTED_COLUMN_MODES = [ const DEFAULT_COLUMN_MODE = COLUMN_MODE_GEOJSON; export default class GeoJsonLayer extends Layer { - declare config: GeoJsonLayerConfig; declare visConfigSettings: GeoJsonVisConfigSettings; + declare config: GeoJsonLayerConfig; declare meta: GeoJsonLayerMeta; declare geoArrowMode: boolean; dataToFeature: GeojsonDataMaps = []; dataContainer: DataContainerInterface | null = null; + filteredIndex: Uint8ClampedArray | null = null; filteredIndexTrigger: number[] | null = null; + centroids: Array = []; _layerInfoModal: { @@ -250,23 +252,28 @@ export default class GeoJsonLayer extends Layer { constructor(props) { super(props); - this.registerVisConfig(geojsonVisConfigs); - this.getPositionAccessor = (dataContainer: DataContainerInterface) => - featureAccessor(this.config.columns)(dataContainer); this._layerInfoModal = { [COLUMN_MODE_TABLE]: GeojsonInfoModalFactory(COLUMN_MODE_TABLE), [COLUMN_MODE_GEOJSON]: GeojsonInfoModalFactory(COLUMN_MODE_GEOJSON) }; + + this.getPositionAccessor = (dataContainer: DataContainerInterface) => + featureAccessor(this.config.columns)(dataContainer); } - get type() { - return GeoJsonLayer.type; + get supportedColumnModes() { + return SUPPORTED_COLUMN_MODES; } + static get type(): 'geojson' { return 'geojson'; } + get type() { + return GeoJsonLayer.type; + } + get name(): 'Polygon' { return 'Polygon'; } @@ -279,10 +286,6 @@ export default class GeoJsonLayer extends Layer { return this.defaultPointColumnPairs; } - get supportedColumnModes() { - return SUPPORTED_COLUMN_MODES; - } - get layerInfoModal() { return { [COLUMN_MODE_GEOJSON]: { @@ -501,7 +504,7 @@ export default class GeoJsonLayer extends Layer { } } - formatLayerData(datasets, oldLayerData) { + formatLayerData(datasets: Datasets, oldLayerData) { if (this.config.dataId === null) { return {}; } @@ -525,12 +528,14 @@ export default class GeoJsonLayer extends Layer { return this.filteredIndex ? this.filteredIndex[d.properties.index] : 1; }; + const getFilterValue = gpuFilter.filterValueAccessor(dataContainer)( + indexAccessor, + filterValueAccessor + ); + return { data, - getFilterValue: gpuFilter.filterValueAccessor(dataContainer)( - indexAccessor, - filterValueAccessor - ), + getFilterValue, getFiltered: isFilteredAccessor, ...accessors }; diff --git a/src/layers/src/geojson-layer/geojson-utils.ts b/src/layers/src/geojson-layer/geojson-utils.ts index ad7e83399e..4c15ee4595 100644 --- a/src/layers/src/geojson-layer/geojson-utils.ts +++ b/src/layers/src/geojson-layer/geojson-utils.ts @@ -88,17 +88,19 @@ export function parseGeoJsonRawFeature(rawFeature: unknown): Feature | null { export function getGeojsonLayerMeta({ dataContainer, getFeature, - config + config, + sortByColumn }: { dataContainer: DataContainerInterface; getFeature: GetFeature; config: LayerBaseConfig; + sortByColumn?: string; }): GeojsonLayerMetaProps { const dataToFeature = config.columnMode === COLUMN_MODE_GEOJSON ? getGeojsonDataMaps(dataContainer, getFeature) : // COLUMN_MODE_TABLE - groupColumnsAsGeoJson(dataContainer, config.columns, 'sortBy'); + groupColumnsAsGeoJson(dataContainer, config.columns, sortByColumn || 'sortBy'); // get bounds from features const bounds = getGeojsonBounds(dataToFeature); diff --git a/src/layers/src/trip-layer/trip-layer.ts b/src/layers/src/trip-layer/trip-layer.ts index 65441a0bb9..8559bbae94 100644 --- a/src/layers/src/trip-layer/trip-layer.ts +++ b/src/layers/src/trip-layer/trip-layer.ts @@ -1,25 +1,32 @@ // SPDX-License-Identifier: MIT // Copyright contributors to the kepler.gl project +import * as arrow from 'apache-arrow'; import memoize from 'lodash.memoize'; import uniq from 'lodash.uniq'; -import Layer, {LayerBaseConfig, defaultGetFieldValue} from '../base-layer'; +import {DATA_TYPES} from 'type-analyzer'; + +import Layer, {LayerBaseConfig, defaultGetFieldValue, LayerBaseConfigPartial} from '../base-layer'; import {TripsLayer as DeckGLTripsLayer} from '@deck.gl/geo-layers'; +import {containValidTime} from '@kepler.gl/common-utils'; import {GEOJSON_FIELDS, PROJECTED_PIXEL_SIZE_MULTIPLIER} from '@kepler.gl/constants'; import TripLayerIcon from './trip-layer-icon'; import { - getGeojsonDataMaps, - getGeojsonBounds, - getGeojsonFeatureTypes, GeojsonDataMaps, detectTableColumns, - groupColumnsAsGeoJson, - applyFiltersToTableColumns + applyFiltersToTableColumns, + getGeojsonLayerMeta, + fieldIsGeoArrow } from '../geojson-layer/geojson-utils'; +import {getGeojsonLayerMetaFromArrow} from '../layer-utils'; -import {isTripGeoJsonField, parseTripGeoJsonTimestamp} from './trip-utils'; +import { + isTripGeoJsonField, + parseTripGeoJsonTimestamp, + parseTripGeoJsonFromGeoArrow +} from './trip-utils'; import TripInfoModalFactory from './trip-info-modal'; import {bisectRight} from 'd3-array'; import { @@ -28,10 +35,17 @@ import { VisConfigColorRange, VisConfigNumber, VisConfigRange, - LayerColumn + LayerColumn, + ProtoDatasetField } from '@kepler.gl/types'; import {default as KeplerTable, Datasets} from '@kepler.gl/table'; -import {DataContainerInterface} from '@kepler.gl/utils'; +import {DataContainerInterface, ArrowDataContainer} from '@kepler.gl/utils'; + +const SUPPORTED_ANALYZER_TYPES = { + [DATA_TYPES.GEOMETRY]: true, + [DATA_TYPES.GEOMETRY_FROM_STRING]: true, + [DATA_TYPES.PAIR_GEOMETRY_FROM_STRING]: true +}; export type TripLayerVisConfigSettings = { opacity: VisConfigNumber; @@ -99,13 +113,21 @@ export const tripVisConfigs: { export const featureAccessor = ({geojson}: TripLayerColumnsConfig) => (dc: DataContainerInterface) => - d => + (d: {index: number}) => dc.valueAt(d.index, geojson.fieldIdx); + export const featureResolver = ({geojson}: TripLayerColumnsConfig) => geojson.fieldIdx; -const getTableModeValueAccessor = f => { + +const geoColumnAccessor = + ({geojson}: TripLayerColumnsConfig) => + (dc: DataContainerInterface): arrow.Vector | null => + dc.getColumn?.(geojson.fieldIdx) as arrow.Vector; + +const getTableModeValueAccessor = feature => { // Called from gpu-filter-utils.getFilterValueAccessor() - return field => f.properties.values.map(v => field.valueAccessor(v)); + return field => feature.properties.values.map(v => field.valueAccessor(v)); }; + const getTableModeFieldValue = (field, data) => { let rv; if (typeof data === 'function') { @@ -116,6 +138,11 @@ const getTableModeFieldValue = (field, data) => { return rv; }; +const geoFieldAccessor = + ({geojson}: TripLayerColumnsConfig) => + (dc: DataContainerInterface): ProtoDatasetField | null => + dc.getField ? dc.getField(geojson.fieldIdx) : null; + export const COLUMN_MODE_GEOJSON = 'geojson'; export const COLUMN_MODE_TABLE = 'table'; const SUPPORTED_COLUMN_MODES = [ @@ -137,25 +164,39 @@ export default class TripLayer extends Layer { declare visConfigSettings: TripLayerVisConfigSettings; declare config: TripLayerConfig; declare meta: TripLayerMeta; - declare dataContainer: DataContainerInterface | null; + declare geoArrowMode: boolean; + + dataToFeature: GeojsonDataMaps = []; + dataContainer: DataContainerInterface | null = null; + + filteredIndex: Uint8ClampedArray | null = null; + filteredIndexTrigger: number[] | null = null; + + dataToTimeStamp: any[] = []; + + _layerInfoModal: { + [COLUMN_MODE_TABLE]: () => React.JSX.Element; + [COLUMN_MODE_GEOJSON]: () => React.JSX.Element; + }; - dataToFeature: GeojsonDataMaps; - dataToTimeStamp: any[]; getFeature: (columns: TripLayerColumnsConfig) => (dataContainer: DataContainerInterface) => any; - _layerInfoModal: Record JSX.Element>; constructor(props) { super(props); - - this.dataToFeature = []; - this.dataToTimeStamp = []; - this.dataContainer = null; this.registerVisConfig(tripVisConfigs); - this.getFeature = memoize(featureAccessor, featureResolver); this._layerInfoModal = { [COLUMN_MODE_TABLE]: TripInfoModalFactory(COLUMN_MODE_TABLE), [COLUMN_MODE_GEOJSON]: TripInfoModalFactory(COLUMN_MODE_GEOJSON) }; + + this.getFeature = memoize(featureAccessor, featureResolver); + + this.getPositionAccessor = (dataContainer: DataContainerInterface) => { + if (this.config.columnMode === COLUMN_MODE_GEOJSON) { + return this.getFeature(this.config.columns)(dataContainer); + } + return null; + }; } get supportedColumnModes() { @@ -165,6 +206,7 @@ export default class TripLayer extends Layer { static get type(): 'trip' { return 'trip'; } + get type() { return TripLayer.type; } @@ -181,6 +223,25 @@ export default class TripLayer extends Layer { return this.defaultPointColumnPairs; } + get layerInfoModal() { + return { + [COLUMN_MODE_GEOJSON]: { + id: 'iconInfo', + template: this._layerInfoModal[COLUMN_MODE_GEOJSON], + modalProps: { + title: 'modal.tripInfo.title' + } + }, + [COLUMN_MODE_TABLE]: { + id: 'iconInfo', + template: this._layerInfoModal[COLUMN_MODE_TABLE], + modalProps: { + title: 'modal.tripInfo.titleTable' + } + } + }; + } + accessVSFieldValue() { if (this.config.columnMode === COLUMN_MODE_GEOJSON) { return defaultGetFieldValue; @@ -190,7 +251,6 @@ export default class TripLayer extends Layer { get visualChannels() { const visualChannels = super.visualChannels; - return { ...visualChannels, color: { @@ -216,47 +276,69 @@ export default class TripLayer extends Layer { return this.config.animation.domain; } - get layerInfoModal() { - return { - [COLUMN_MODE_GEOJSON]: { - id: 'iconInfo', - template: this._layerInfoModal[COLUMN_MODE_GEOJSON], - modalProps: { - title: 'modal.tripInfo.title' - } - }, - [COLUMN_MODE_TABLE]: { - id: 'iconInfo', - template: this._layerInfoModal[COLUMN_MODE_TABLE], - modalProps: { - title: 'modal.tripInfo.titleTable' - } + updateAnimationDomain(domain) { + this.updateLayerConfig({ + animation: { + ...this.config.animation, + domain } - }; - } - - getPositionAccessor(dataContainer: DataContainerInterface) { - if (this.config.columnMode === COLUMN_MODE_GEOJSON) { - return this.getFeature(this.config.columns)(dataContainer); - } - return null; + }); } static findDefaultLayerProps( {label, fields = [], dataContainer, id}: KeplerTable, foundLayers: any[] ) { - const geojsonColumns = fields.filter(f => f.type === 'geojson').map(f => f.name); + const geojsonColumns = fields + .filter( + f => + (f.type === 'geojson' || f.type === 'geoarrow') && + f.analyzerType && + SUPPORTED_ANALYZER_TYPES[f.analyzerType] + ) + .map(f => f.name); const defaultColumns = { geojson: uniq([...GEOJSON_FIELDS.geojson, ...geojsonColumns]) }; - const geoJsonColumns = this.findDefaultColumnField(defaultColumns, fields); + const foundColumns = this.findDefaultColumnField(defaultColumns, fields); - const tripGeojsonColumns = (geoJsonColumns || []).filter(col => - isTripGeoJsonField(dataContainer, fields[col.geojson.fieldIdx]) - ); + const tripGeojsonColumns = (foundColumns || []).filter(col => { + const geoField = fields[col.geojson.fieldIdx]; + if (fieldIsGeoArrow(geoField)) { + const geoColumn = geoColumnAccessor(col)(dataContainer); + if (!geoColumn) return false; + + // ! query only small sample of features + const info = getGeojsonLayerMetaFromArrow({ + dataContainer, + geoColumn, + geoField, + chunkIndex: 0 + }); + + // TODO use common check logic from trip-utils + const NUM_DIMENSIONS_FOR_TRIPS = 3; // TODO should check for 4 + if ( + info.featureTypes.line && + // @ts-expect-error + info.dataToFeature?.[0]?.lines?.positions.size === NUM_DIMENSIONS_FOR_TRIPS + ) { + // @ts-expect-error + const values = info.dataToFeature[0].lines.positions.value; + const tsHolder: number[] = []; + values.forEach((value, index) => { + if (index % NUM_DIMENSIONS_FOR_TRIPS === NUM_DIMENSIONS_FOR_TRIPS - 1) + tsHolder.push(value); + }); + return Boolean(containValidTime(tsHolder as any[])); + } + } else { + return isTripGeoJsonField(dataContainer, fields[col.geojson.fieldIdx]); + } + return false; + }); if (tripGeojsonColumns.length) { return { @@ -280,9 +362,10 @@ export default class TripLayer extends Layer { return {props: []}; } - getDefaultLayerConfig(props) { + getDefaultLayerConfig(props: LayerBaseConfigPartial) { + const defaultLayerConfig = super.getDefaultLayerConfig(props ?? {}); return { - ...super.getDefaultLayerConfig(props), + ...defaultLayerConfig, columnMode: props?.columnMode ?? DEFAULT_COLUMN_MODE, animation: { enabled: true, @@ -301,6 +384,34 @@ export default class TripLayer extends Layer { } calculateDataAttribute(dataset: KeplerTable) { + this.geoArrowMode = fieldIsGeoArrow( + geoFieldAccessor(this.config.columns)(dataset.dataContainer) + ); + + const {dataContainer, filteredIndex} = dataset; + if (this.geoArrowMode) { + // filter geojson/arrow table by values and make a partial copy of the raw table is expensive + // so we will use filteredIndex to create an attribute e.g. filteredIndex [0|1] for GPU filtering + // in deck.gl layer, see: FilterArrowExtension in @kepler.gl/deckgl-layers + if (!this.filteredIndex || this.filteredIndex.length !== dataContainer.numRows()) { + // for incremental data loading, we need to update filteredIndex + this.filteredIndex = new Uint8ClampedArray(dataContainer.numRows()); + this.filteredIndex.fill(1); + } + + // check if filteredIndex is a range from 0 to numRows if it is, we don't need to update it + const isRange = filteredIndex && filteredIndex.length === dataContainer.numRows(); + if (!isRange || this.filteredIndexTrigger !== null) { + this.filteredIndex.fill(0); + for (let i = 0; i < filteredIndex.length; ++i) { + this.filteredIndex[filteredIndex[i]] = 1; + } + this.filteredIndexTrigger = filteredIndex; + } + // for arrow, always return full dataToFeature instead of a filtered one, so there is no need to update attributes in GPU + return this.dataToFeature; + } + switch (this.config.columnMode) { case COLUMN_MODE_GEOJSON: { return ( @@ -328,70 +439,107 @@ export default class TripLayer extends Layer { const {dataContainer, gpuFilter} = datasets[this.config.dataId]; const {data} = this.updateData(datasets, oldLayerData); - let valueAccessor; + let filterValueAccessor; if (this.config.columnMode === COLUMN_MODE_GEOJSON) { - valueAccessor = (dc: DataContainerInterface, f, fieldIndex: number) => { + filterValueAccessor = (dc: DataContainerInterface, f, fieldIndex: number) => { return dc.valueAt(f.properties.index, fieldIndex); }; } else { - valueAccessor = getTableModeValueAccessor; + filterValueAccessor = getTableModeValueAccessor; } const indexAccessor = f => f.properties.index; const dataAccessor = () => d => ({index: d.properties.index}); const accessors = this.getAttributeAccessors({dataAccessor, dataContainer}); + + const isFilteredAccessor = d => { + return this.filteredIndex ? this.filteredIndex[d.properties.index] : 1; + }; + const getFilterValue = gpuFilter.filterValueAccessor(dataContainer)( indexAccessor, - valueAccessor + filterValueAccessor ); return { data, getFilterValue, + getFiltered: isFilteredAccessor, getPath: d => d.geometry.coordinates, getTimestamps: d => this.dataToTimeStamp[d.properties.index], ...accessors }; } - updateAnimationDomain(domain) { - this.updateLayerConfig({ - animation: { - ...this.config.animation, - domain - } - }); - } - updateLayerMeta(dataset: KeplerTable) { const {dataContainer} = dataset; + + this.dataContainer = dataContainer; + + this.geoArrowMode = fieldIsGeoArrow( + geoFieldAccessor(this.config.columns)(dataset.dataContainer) + ); + let getFeature; - if (this.config.columnMode === COLUMN_MODE_GEOJSON) { + + if (this.geoArrowMode && dataContainer instanceof ArrowDataContainer) { + const geoColumn = geoColumnAccessor(this.config.columns)(dataContainer); + const geoField = geoFieldAccessor(this.config.columns)(dataContainer); + + // update the latest batch/chunk of geoarrow data when loading data incrementally + if (geoColumn && geoField && this.dataToFeature.length < dataContainer.numChunks()) { + // for incrementally loading data, we only load and render the latest batch; otherwise, we will load and render all batches + const isIncrementalLoad = dataContainer.numChunks() - this.dataToFeature.length === 1; + + // TODO ! sortBy 'timestamp' + const {dataToFeature, bounds, fixedRadius, featureTypes} = getGeojsonLayerMetaFromArrow({ + dataContainer, + geoColumn, + geoField, + ...(isIncrementalLoad ? {chunkIndex: this.dataToFeature.length} : null) + }); + // if (centroids) this.centroids = this.centroids.concat(centroids); + + // TODO separate binary and json features + this.dataToFeature = [...this.dataToFeature, ...dataToFeature]; + + const {dataToFeatureOut, dataToTimeStamp, animationDomain} = parseTripGeoJsonFromGeoArrow( + // @ts-expect-error + this.dataToFeature + ); + + // @ts-expect-error + this.dataToFeature = dataToFeatureOut; + this.dataToTimeStamp = dataToTimeStamp; + this.updateAnimationDomain(animationDomain); + + this.updateMeta({bounds, fixedRadius, featureTypes}); + } + } else if (this.dataToFeature.length === 0) { getFeature = this.getPositionAccessor(dataContainer); if (getFeature === this.meta.getFeature) { // TODO: revisit this after gpu filtering return; } - this.dataToFeature = getGeojsonDataMaps(dataContainer, getFeature); - } else { - this.dataContainer = dataContainer; - this.dataToFeature = groupColumnsAsGeoJson(dataContainer, this.config.columns, 'timestamp'); - } - const {dataToTimeStamp, animationDomain} = parseTripGeoJsonTimestamp(this.dataToFeature); + const {dataToFeature, bounds, featureTypes} = getGeojsonLayerMeta({ + dataContainer, + getFeature, + config: this.config, + sortByColumn: 'timestamp' + }); + // if (centroids) this.centroids = centroids; + this.dataToFeature = dataToFeature; - this.dataToTimeStamp = dataToTimeStamp; - this.updateAnimationDomain(animationDomain); + const {dataToTimeStamp, animationDomain} = parseTripGeoJsonTimestamp(this.dataToFeature); - // get bounds from features - const bounds = getGeojsonBounds(this.dataToFeature); + this.dataToTimeStamp = dataToTimeStamp; + this.updateAnimationDomain(animationDomain); - // keep a record of what type of geometry the collection has - const featureTypes = getGeojsonFeatureTypes(this.dataToFeature); - - this.updateMeta({bounds, featureTypes, getFeature}); + this.updateMeta({bounds, featureTypes, getFeature}); + } } - setInitialLayerConfig(dataset) { + setInitialLayerConfig(dataset: KeplerTable) { const {dataContainer} = dataset; if (!dataContainer.numRows()) { return this; @@ -401,7 +549,7 @@ export default class TripLayer extends Layer { // if not found, we try to set it to id / lat /lng /ts if (!this.config.columns.geojson.value) { // find columns from lat, lng, id, and ts - const columnConfig = detectTableColumns(dataset, this.config.columns); + const columnConfig = detectTableColumns(dataset, this.config.columns, 'timestamp'); if (columnConfig) { this.updateLayerConfig({ ...columnConfig, @@ -411,6 +559,7 @@ export default class TripLayer extends Layer { } this.updateLayerMeta(dataset); + return this; } diff --git a/src/layers/src/trip-layer/trip-utils.ts b/src/layers/src/trip-layer/trip-utils.ts index 522905cb6e..29cf24dd16 100644 --- a/src/layers/src/trip-layer/trip-utils.ts +++ b/src/layers/src/trip-layer/trip-utils.ts @@ -4,9 +4,13 @@ import {parseGeoJsonRawFeature, getGeojsonFeatureTypes} from '../geojson-layer/geojson-utils'; import {DataContainerInterface, getSampleContainerData, timeToUnixMilli} from '@kepler.gl/utils'; import {containValidTime, notNullorUndefined} from '@kepler.gl/common-utils'; +import {BinaryFeatureCollection} from '@loaders.gl/schema'; import {Feature} from '@turf/helpers'; import {GeoJsonProperties, Geometry} from 'geojson'; +// TODO: We should check for 4 ! +const NUM_DIMENSIONS_FOR_TRIPS = 3; + /** * Parse geojson from string * @param {array} samples feature object values @@ -103,6 +107,92 @@ export function parseTripGeoJsonTimestamp(dataToFeature: any[]) { return {dataToTimeStamp, animationDomain}; } +/** + * Converts Deck.gl's BinaryFeatureCollection into regular json features supported by TripLayer. + * TODO: investigate whether BinaryFeatureCollection can be easily passed to Trips Layer as attributes. + * Info for (sublayer) Deck.gl's PathsLayer: https://deck.gl/docs/api-reference/layers/path-layer#use-binary-attributes + * @param dataToFeature An array of binary feature collections. Only lines are checked. + * @returns TripLayer-compatible features and timestamps. + */ +export function parseTripGeoJsonFromGeoArrow(dataToFeature: BinaryFeatureCollection[]): { + dataToTimeStamp: number[][]; + dataToFeatureOut: Feature[]; + animationDomain: [number, number] | null; +} { + // Analyze type based on coordinates of the 1st lineString + // select a sample trip to analyze time format + const empty = {dataToTimeStamp: [], dataToFeatureOut: [], animationDomain: null}; + + // We need 4 dimensions + const lines = dataToFeature[0].lines; + if (!lines || lines.positions?.size !== NUM_DIMENSIONS_FOR_TRIPS) { + return empty; + } + + const positions = lines.positions.value; + const pathIndices = lines.pathIndices.value; + + // Check for proper type fo the time values + const timestampsSample: number[] = []; + for (let i = NUM_DIMENSIONS_FOR_TRIPS - 1; i < positions.length; i += NUM_DIMENSIONS_FOR_TRIPS) { + timestampsSample.push(positions[i]); + } + const analyzedType = containValidTime( + timestampsSample as any[] /* why only strings are expected? */ + ); + if (!analyzedType) { + return empty; + } + const {format} = analyzedType; + const getTimeValue = value => { + return value && notNullorUndefined(value) ? timeToUnixMilli(value, format) : null; + }; + + // Transform binary buffers to standard Features and separate timestamp data. + const dataToFeatureOut: Feature[] = []; + const dataToTimeStamp: number[][] = []; + lines.properties.forEach((f, featureIndex) => { + // get number of coordinates in current path + const prevIndex = pathIndices[featureIndex]; + const numCoordinates = pathIndices[featureIndex + 1] - prevIndex; + const baseShift = prevIndex * NUM_DIMENSIONS_FOR_TRIPS; + + const coordinates: number[][] = []; + const timeValues: number[] = []; + for (let coordIndex = 0; coordIndex < numCoordinates; ++coordIndex) { + const baseIndex = baseShift + coordIndex * NUM_DIMENSIONS_FOR_TRIPS; + + // TODO add elevation, this was used for testing with DuckDB + const coordinate = [positions[baseIndex], positions[baseIndex + 1], 0]; + + const timeValue = positions[baseIndex + NUM_DIMENSIONS_FOR_TRIPS - 1]; + const parsedTimeValue = getTimeValue(timeValue); + + if (parsedTimeValue) { + coordinates.push(coordinate); + timeValues.push(parsedTimeValue); + } + } + + dataToFeatureOut.push({ + type: 'Feature', + geometry: { + type: 'LineString', + coordinates + }, + properties: { + index: featureIndex + } + }); + + dataToTimeStamp.push(timeValues); + }); + + const animationDomain = getAnimationDomainFromTimestamps(dataToTimeStamp); + + return {dataToTimeStamp, dataToFeatureOut, animationDomain}; +} + function findMinFromSorted(list: number[]) { // check if list is null since the default value [] will only be applied when the param is undefined return list?.find(d => notNullorUndefined(d) && Number.isFinite(d)) || null;