feat: add UTC export by default for devices and assets (#342)

* feat: add UTC export by default for devices and assets --------- Co-authored-by: Olivier Dumas <[email protected]>
kuzzleio · Jul 11, 2024 · 878ca58 · 878ca58
1 parent 662e642
commit 878ca58
Show file tree

Hide file tree

Showing 7 changed files with 41 additions and 8 deletions.
diff --git a/lib/modules/measure/MeasureExporter.ts b/lib/modules/measure/MeasureExporter.ts
@@ -124,7 +124,7 @@ export class MeasureExporter extends AbstractExporter<MeasureExportParams> {
     columns: Array<Column & { shouldCheckName: boolean; name: string }>,
     hit: KHit<MeasureContent>,
   ) {
-    return columns.map(({ name, shouldCheckName, path }) => {
+    return columns.map(({ name, shouldCheckName, path, isIsoDate }) => {
       if (
         shouldCheckName &&
         this.target === InternalCollection.ASSETS &&
@@ -133,6 +133,11 @@ export class MeasureExporter extends AbstractExporter<MeasureExportParams> {
         return null;
       }
 
+      const formattedValue = _.get(hit, path, null);
+      if (formattedValue !== null && isIsoDate) {
+        return new Date(formattedValue).toISOString();
+      }
+
       return _.get(hit, path, null);
     });
   }
@@ -195,6 +200,11 @@ export class MeasureExporter extends AbstractExporter<MeasureExportParams> {
     const columns: Column[] = [
       { header: "Measure Id", path: "_id" },
       { header: "Measured At", path: "_source.measuredAt" },
+      {
+        header: "Measured At ISO",
+        isIsoDate: true,
+        path: "_source.measuredAt",
+      },
       { header: "Measure Type", path: "_source.type" },
       { header: "Device Id", path: "_source.origin._id" },
       { header: "Device Model", path: "_source.origin.deviceModel" },

diff --git a/lib/modules/shared/services/AbstractExporter.ts b/lib/modules/shared/services/AbstractExporter.ts
@@ -31,6 +31,7 @@ export interface Column {
   header: string;
   path: string;
   isMeasure?: boolean;
+  isIsoDate?: boolean;
 }
 
 export abstract class AbstractExporter<P extends ExportParams = ExportParams> {
@@ -117,7 +118,13 @@ export abstract class AbstractExporter<P extends ExportParams = ExportParams> {
     columns: Column[],
     hit: KHit<KDocumentContentGeneric>,
   ): string[] {
-    return columns.map(({ path }) => _.get(hit, path, null));
+    return columns.map(({ path, isIsoDate }) => {
+      const formattedValue = _.get(hit, path, null);
+      if (formattedValue !== null && isIsoDate) {
+        return new Date(formattedValue).toISOString();
+      }
+      return _.get(hit, path, null);
+    });
   }
 
   async getExport(engineId: string, exportId: string): Promise<P> {
@@ -141,7 +148,6 @@ export abstract class AbstractExporter<P extends ExportParams = ExportParams> {
     let result = request;
     try {
       stream.write(stringify([columns.map((column) => column.header)]));
-
       while (result) {
         for (const hit of result.hits) {
           stream.write(stringify([this.formatHit(columns, hit)]));

diff --git a/lib/modules/shared/services/DigitalTwinExporter.ts b/lib/modules/shared/services/DigitalTwinExporter.ts
@@ -47,9 +47,15 @@ export class DigitalTwinExporter extends AbstractExporter {
       { header: "Reference", path: "_source.reference" },
       ...measureColumns,
       { header: "lastMeasuredAt", path: "_source.lastMeasuredAt" },
+      {
+        header: "lastMeasuredAtISO",
+        isIsoDate: true,
+        path: "_source.lastMeasuredAt",
+      },
     ];
 
     const stream = this.getExportStream(digitalTwins, columns);
+
     await this.sdk.ms.del(this.exportRedisKey(engineId, exportId));
 
     return stream;

diff --git a/tests/scenario/modules/assets/action-export-measures.test.ts b/tests/scenario/modules/assets/action-export-measures.test.ts
@@ -114,11 +114,12 @@ describe("AssetsController:exportMeasures", () => {
 
     expect(csv).toHaveLength(5);
     expect(csv[0]).toBe(
-      "Measure Id,Measured At,Measure Type,Device Id,Device Model,Asset Id,Asset Model,temperatureExt.temperature,temperatureInt.temperature,position.position,position.accuracy,position.altitude,temperatureWeather.temperature\n",
+      "Measure Id,Measured At,Measured At ISO,Measure Type,Device Id,Device Model,Asset Id,Asset Model,temperatureExt.temperature,temperatureInt.temperature,position.position,position.accuracy,position.altitude,temperatureWeather.temperature\n",
     );
     const [
       payloadId,
       measuredAt,
+      measuredAtISO,
       measureType,
       deviceId,
       deviceModel,
@@ -132,6 +133,7 @@ describe("AssetsController:exportMeasures", () => {
 
     expect(typeof payloadId).toBe("string");
     expect(typeof parseFloat(measuredAt)).toBe("number");
+    expect(typeof measuredAtISO).toBe("string");
     expect(measureType).toBe("temperature");
     expect(deviceId).toBe("DummyTempPosition-linked2");
     expect(deviceModel).toBe("DummyTempPosition");

diff --git a/tests/scenario/modules/assets/action-export.test.ts b/tests/scenario/modules/assets/action-export.test.ts
@@ -23,6 +23,7 @@ function getExportedColums(row) {
     temperatureInt: parsedRow[6],
     temperatureWeather: parsedRow[7],
     lastMeasuredAt: parsedRow[8],
+    lastMeasuredAtISO: parsedRow[9],
   };
 }
 
@@ -78,7 +79,7 @@ describe("AssetsController:exportMeasures", () => {
     writeFileSync("./assets.csv", csv.join(""));
 
     expect(csv[0]).toBe(
-      "Model,Reference,position,position.accuracy,position.altitude,temperatureExt,temperatureInt,temperatureWeather,lastMeasuredAt\n",
+      "Model,Reference,position,position.accuracy,position.altitude,temperatureExt,temperatureInt,temperatureWeather,lastMeasuredAt,lastMeasuredAtISO\n",
     );
 
     expect(csv).toHaveLength(assetCount + 1);
@@ -94,6 +95,7 @@ describe("AssetsController:exportMeasures", () => {
     expect(typeof parseFloat(row1.temperatureInt)).toBe("number");
     expect(typeof parseFloat(row1.temperatureWeather)).toBe("number");
     expect(typeof parseFloat(row1.lastMeasuredAt)).toBe("number");
+    expect(typeof row1.lastMeasuredAtISO).toBe("string");
 
     const row2 = getExportedColums(csv[2]);
 
@@ -106,5 +108,6 @@ describe("AssetsController:exportMeasures", () => {
     expect(typeof parseFloat(row2.temperatureInt)).toBe("number");
     expect(typeof parseFloat(row2.temperatureWeather)).toBe("number");
     expect(typeof parseFloat(row2.lastMeasuredAt)).toBe("number");
+    expect(typeof row2.lastMeasuredAtISO).toBe("string");
   });
 });
diff --git a/tests/scenario/modules/devices/action-export-measures.test.ts b/tests/scenario/modules/devices/action-export-measures.test.ts
@@ -131,12 +131,13 @@ describe("DevicesController:exportMeasures", () => {
 
     expect(csv).toHaveLength(26);
     expect(csv[0]).toBe(
-      "Measure Id,Measured At,Measure Type,Device Id,Device Model,Asset Id,Asset Model,temperature.temperature,accelerationSensor.acceleration.x,accelerationSensor.acceleration.y,accelerationSensor.acceleration.z,accelerationSensor.accuracy,battery.battery\n",
+      "Measure Id,Measured At,Measured At ISO,Measure Type,Device Id,Device Model,Asset Id,Asset Model,temperature.temperature,accelerationSensor.acceleration.x,accelerationSensor.acceleration.y,accelerationSensor.acceleration.z,accelerationSensor.accuracy,battery.battery\n",
     );
 
     const [
       payloadId,
       measuredAt,
+      measuredAtISO,
       tempMeasureType,
       deviceId,
       deviceModel,
@@ -145,6 +146,7 @@ describe("DevicesController:exportMeasures", () => {
       temperature,
     ] = csvParse(csv[1])[0];
     const [
+      ,
       ,
       ,
       accMeasureType,
@@ -161,6 +163,7 @@ describe("DevicesController:exportMeasures", () => {
 
     expect(typeof payloadId).toBe("string");
     expect(typeof parseFloat(measuredAt)).toBe("number");
+    expect(typeof measuredAtISO).toBe("string");
     expect(tempMeasureType).toBe("temperature");
     expect(deviceId).toBe("DummyTemp-linked1");
     expect(deviceModel).toBe("DummyTemp");
@@ -203,7 +206,7 @@ describe("DevicesController:exportMeasures", () => {
 
     expect(csv).toHaveLength(3);
     expect(csv[0]).toBe(
-      "Measure Id,Measured At,Measure Type,Device Id,Device Model,Asset Id,Asset Model,temperature.temperature,accelerationSensor.acceleration.x,accelerationSensor.acceleration.y,accelerationSensor.acceleration.z,accelerationSensor.accuracy,battery.battery\n",
+      "Measure Id,Measured At,Measured At ISO,Measure Type,Device Id,Device Model,Asset Id,Asset Model,temperature.temperature,accelerationSensor.acceleration.x,accelerationSensor.acceleration.y,accelerationSensor.acceleration.z,accelerationSensor.accuracy,battery.battery\n",
     );
   });
 });
diff --git a/tests/scenario/modules/devices/action-export.test.ts b/tests/scenario/modules/devices/action-export.test.ts
@@ -32,6 +32,7 @@ function getExportedColums(row) {
     positionAltitude: parsedRow[9],
     temperature: parsedRow[10],
     lastMeasuredAt: parsedRow[11],
+    lastMeasuredAtISO: parsedRow[12],
   };
 }
 
@@ -86,7 +87,7 @@ describe("AssetsController:exportMeasures", () => {
     });
 
     expect(csv[0]).toBe(
-      "Model,Reference,accelerationSensor.x,accelerationSensor.y,accelerationSensor.z,accelerationSensor.accuracy,battery,position,position.accuracy,position.altitude,temperature,lastMeasuredAt\n",
+      "Model,Reference,accelerationSensor.x,accelerationSensor.y,accelerationSensor.z,accelerationSensor.accuracy,battery,position,position.accuracy,position.altitude,temperature,lastMeasuredAt,lastMeasuredAtISO\n",
     );
 
     expect(csv).toHaveLength(deviceCount + 1);
@@ -105,6 +106,7 @@ describe("AssetsController:exportMeasures", () => {
     expect(typeof parseFloat(row1.positionAltitude)).toBe("number");
     expect(typeof parseFloat(row1.temperature)).toBe("number");
     expect(typeof parseFloat(row1.lastMeasuredAt)).toBe("number");
+    expect(typeof row1.lastMeasuredAtISO).toBe("string");
 
     const row2 = getExportedColums(csv[1]);
 
@@ -120,5 +122,6 @@ describe("AssetsController:exportMeasures", () => {
     expect(typeof parseFloat(row2.positionAltitude)).toBe("number");
     expect(typeof parseFloat(row2.temperature)).toBe("number");
     expect(typeof parseFloat(row2.lastMeasuredAt)).toBe("number");
+    expect(typeof row2.lastMeasuredAtISO).toBe("string");
   });
 });