Fix composite measures should be correctly exported to CSV

lib/modules/measure/MeasureExporter.ts

@@ -1,20 +1,25 @@
 import { randomUUID } from "node:crypto";
 import { PassThrough } from "node:stream";
 
-import _ from "lodash";
+import { stringify } from "csv-stringify/sync";
 import {
   InternalError,
   JSONObject,
   KDocument,
   NotFoundError,
   User,
 } from "kuzzle";
-import { stringify } from "csv-stringify/sync";
+import _ from "lodash";
 
-import { AskModelAssetGet, AskModelDeviceGet } from "../model";
+import {
+  AskModelAssetGet,
+  AskModelDeviceGet,
+  AskModelMeasureGet,
+} from "../model";
 import { DeviceManagerPlugin, InternalCollection } from "../plugin";
-import { ask, DigitalTwinContent } from "../shared";
+import { DigitalTwinContent, ask, flattenObject } from "../shared";
 
+import { NamedMeasures } from "../decoder";
 import { MeasureContent } from "./exports";
 
 type ExportOptions = {
@@ -177,6 +182,10 @@ export class MeasureExporter {
         }
       );
 
+      const measureColumns = await this.generateMeasureColumns(
+        modelDocument[target].measures
+      );
+
       const columns: Column[] = [
         { header: "Payload Id", path: "_id" },
         { header: "Measured At", path: "_source.measuredAt" },
@@ -185,11 +194,7 @@ export class MeasureExporter {
         { header: "Device Model", path: "_source.origin.deviceModel" },
         { header: "Asset Id", path: "_source.asset._id" },
         { header: "Asset Model", path: "_source.asset.model" },
-        ...modelDocument[target].measures.map((measure) => ({
-          header: measure.name,
-          isMeasure: true,
-          path: `_source.values.${measure.type}`,
-        })),
+        ...measureColumns,
       ];
 
       stream.write(stringify([columns.map((column) => column.header)]));
@@ -224,6 +229,66 @@ export class MeasureExporter {
     }
   }
 
+  private async generateMeasureColumns(
+    documentMeasures: NamedMeasures
+  ): Promise<Array<Column & { isMeasure: true }>> {
+    /**
+     * @example
+     * {
+     *   temperature: ['temperature.type'],
+     *   acceleration: [
+     *     'acceleration.properties.x.type',
+     *     'acceleration.properties.y.type',
+     *     'acceleration.properties.z.type',
+     *     'accuracy.type'
+     *   ]
+     * }
+     */
+    const mappingsByMeasureType: Record<string, string[]> = {};
+    const measures: Array<Column & { isMeasure: true }> = [];
+
+    for (const { name, type } of documentMeasures) {
+      if (!(type in mappingsByMeasureType)) {
+        const { measure } = await ask<AskModelMeasureGet>(
+          "ask:device-manager:model:measure:get",
+          { type }
+        );
+
+        mappingsByMeasureType[type] = Object.keys(
+          flattenObject(measure.valuesMappings)
+        );
+      }
+
+      for (const measure of mappingsByMeasureType[type]) {
+        /**
+         * @example
+         * 'temperature', 'acceleration.x', 'accuracy'
+         */
+        const path = measure.replace(".type", "").replace(".properties", "");
+        /**
+         * @example
+         * '', 'x', 'accuracy'
+         */
+        const pathWithoutType = path.replace(new RegExp(type + "[.]?"), "");
+
+        /**
+         * @example
+         * 'temperature', 'acceleration.x', 'acceleration.accuracy'
+         */
+        const header = `${name}${
+          pathWithoutType !== "" ? `.${pathWithoutType}` : ""
+        }`;
+        measures.push({
+          header,
+          isMeasure: true,
+          path: `_source.values.${path}`,
+        });
+      }
+    }
+
+    return measures;
+  }
+
   async getExport(exportId: string) {
     const exportParams = await this.sdk.ms.get(this.exportRedisKey(exportId));
 

lib/modules/measure/measures/PositionMeasure.ts

@@ -13,8 +13,8 @@ export type PositionMeasurement = {
 
 export const positionMeasureDefinition: MeasureDefinition = {
   valuesMappings: {
+    position: { type: "geo_point" },
     accuracy: { type: "float" },
     altitude: { type: "float" },
-    position: { type: "geo_point" },
   },
 };

lib/modules/model/ModelService.ts

@@ -22,7 +22,11 @@ import {
   DeviceModelContent,
   MeasureModelContent,
 } from "./types/ModelContent";
-import { AskModelAssetGet, AskModelDeviceGet } from "./types/ModelEvents";
+import {
+  AskModelAssetGet,
+  AskModelDeviceGet,
+  AskModelMeasureGet,
+} from "./types/ModelEvents";
 
 export class ModelService {
   private config: DeviceManagerConfiguration;
@@ -56,6 +60,14 @@ export class ModelService {
         return deviceModel._source;
       }
     );
+    onAsk<AskModelMeasureGet>(
+      "ask:device-manager:model:measure:get",
+      async ({ type }) => {
+        const measureModel = await this.getMeasure(type);
+
+        return measureModel._source;
+      }
+    );
   }
 
   async writeAsset(

lib/modules/model/types/ModelEvents.ts

@@ -1,4 +1,8 @@
-import { AssetModelContent, DeviceModelContent } from "./ModelContent";
+import {
+  AssetModelContent,
+  DeviceModelContent,
+  MeasureModelContent,
+} from "./ModelContent";
 
 export type AskModelAssetGet = {
   name: "ask:device-manager:model:asset:get";
@@ -15,3 +19,11 @@ export type AskModelDeviceGet = {
 
   result: DeviceModelContent;
 };
+
+export type AskModelMeasureGet = {
+  name: "ask:device-manager:model:measure:get";
+
+  payload: { type: string };
+
+  result: MeasureModelContent;
+};

tests/application/app.ts

@@ -1,15 +1,15 @@
 import util from "node:util";
 
-import { Backend, HttpStream, KuzzleRequest } from "kuzzle";
+import { Backend, KuzzleRequest } from "kuzzle";
 
 import { DeviceManagerPlugin } from "../../index";
 
-import { DummyTempDecoder, DummyTempPositionDecoder } from "./decoders";
-import { registerTestPipes } from "./tests/pipes";
-import { TestsController } from "./tests/controller";
 import { containerAssetDefinition } from "./assets/Container";
 import { warehouseAssetDefinition } from "./assets/Warehouse";
-import { PassThrough } from "node:stream";
+import { DummyTempDecoder, DummyTempPositionDecoder } from "./decoders";
+import { TestsController } from "./tests/controller";
+import { registerTestPipes } from "./tests/pipes";
+import { accelerationMeasureDefinition } from "./measures/AccelerationMeasure";
 
 const app = new Backend("kuzzle");
 
@@ -43,13 +43,7 @@ deviceManager.models.registerAsset(
   warehouseAssetDefinition
 );
 
-deviceManager.models.registerMeasure("acceleration", {
-  valuesMappings: {
-    x: { type: "float" },
-    y: { type: "float" },
-    z: { type: "float" },
-  },
-});
+deviceManager.models.registerMeasure("acceleration", accelerationMeasureDefinition);
 
 registerTestPipes(app);
 
@@ -70,29 +64,6 @@ app.config.content.plugins["kuzzle-plugin-logger"].services.stdout.level =
 // @ts-ignore
 app.config.content.limits.documentsWriteCount = 5000;
 
-let searchQuery;
-
-async function sendResult(stream, searchQuery) {
-  let result = await app.sdk.document.search(
-    "device-manager",
-    "payloads",
-    {
-      query: searchQuery,
-    },
-    { scroll: "5s" }
-  );
-
-  while (result) {
-    for (const hit of result.hits) {
-      stream.write(JSON.stringify(hit));
-    }
-
-    result = await result.next();
-  }
-
-  stream.end();
-}
-
 app
   .start()
   .then(() => {

tests/application/decoders/DummyTempDecoder.ts

@@ -7,10 +7,12 @@ import {
   TemperatureMeasurement,
   BatteryMeasurement,
 } from "../../../index";
+import { AccelerationMeasurement } from "../measures/AccelerationMeasure";
 
 export class DummyTempDecoder extends Decoder {
   public measures = [
     { name: "temperature", type: "temperature" },
+    { name: "acceleration", type: "acceleration" },
     { name: "battery", type: "battery" },
   ] as const;
 
@@ -73,6 +75,25 @@ export class DummyTempDecoder extends Decoder {
       }
     );
 
+    if (payload.acceleration !== undefined) {
+      decodedPayload.addMeasurement<AccelerationMeasurement>(
+        payload.deviceEUI,
+        "acceleration",
+        {
+          measuredAt: payload.measuredAt || Date.now(),
+          type: "acceleration",
+          values: {
+            acceleration: {
+              x: payload.acceleration.x,
+              y: payload.acceleration.y,
+              z: payload.acceleration.z,
+            },
+            accuracy: payload.acceleration.accuracy,
+          },
+        }
+      );
+    }
+
     decodedPayload.addMeasurement<BatteryMeasurement>(
       payload.deviceEUI,
       "battery",

tests/application/measures/AccelerationMeasure.ts

@@ -0,0 +1,25 @@
+import { MeasureDefinition } from "lib/modules/measure";
+
+/* eslint-disable sort-keys */
+
+export type AccelerationMeasurement = {
+  acceleration: {
+    x: number;
+    y: number;
+    z: number;
+  };
+  accuracy: number;
+};
+
+export const accelerationMeasureDefinition: MeasureDefinition = {
+  valuesMappings: {
+    acceleration: {
+      properties: {
+        x: { type: "float" },
+        y: { type: "float" },
+        z: { type: "float" },
+      },
+    },
+    accuracy: { type: "float" },
+  },
+};

tests/scenario/migrated/model-controller.test.ts

@@ -548,6 +548,7 @@ describe("features/Model/Controller", () => {
           metadataMappings: { color: { type: "keyword" } },
           measures: [
             { name: "temperature", type: "temperature" },
+            { name: "acceleration", type: "acceleration" },
             { name: "battery", type: "battery" },
           ],
         },

tests/scenario/modules/assets/action-export-measures.test.ts

@@ -10,7 +10,7 @@ jest.setTimeout(10000);
 describe("AssetsController:exportMeasures", () => {
   const sdk = setupHooks();
 
-  it("should prepare export of position measures and return a CSV as stream", async () => {
+  it("should prepare export of temperature measures and return a CSV as stream", async () => {
     await sendPayloads(sdk, "dummy-temp-position", [
       {
         deviceEUI: "linked2",
@@ -67,7 +67,7 @@ describe("AssetsController:exportMeasures", () => {
 
     expect(csv).toHaveLength(5);
     expect(csv[0]).toBe(
-      "Payload Id,Measured At,Measure Type,Device Id,Device Model,Asset Id,Asset Model,temperatureExt,temperatureInt,position,temperatureWeather\n"
+      "Payload Id,Measured At,Measure Type,Device Id,Device Model,Asset Id,Asset Model,temperatureExt,temperatureInt,position,position.accuracy,position.altitude,temperatureWeather\n"
     );
     const [
       payloadId,