feat(configuration): allow to specify Elasticsearch index settings

doc/2/concepts/configuration/index.md

@@ -0,0 +1,198 @@
+---
+code: false
+type: page
+title: Configuration
+description: Plugin configuration
+---
+
+# Configuration
+
+You can customize the behavior of the Device Manager plugin by providing a configuration object in the `plugins` section using the Kuzzle configuration file ([`kuzzlerc`](https://docs.kuzzle.io/core/2/guides/advanced/configuration/)).
+
+
+```js
+{
+  "plugins": {
+    "device-manager": {
+      // Configuration options
+    }
+  }
+}
+```
+
+# Options
+
+## Global
+
+The following global options are available:
+
+| Name                  | Type    | Default          | Description                                                                                        |
+| --------------------- | ------- | ---------------- | -------------------------------------------------------------------------------------------------- |
+| `ignoreStartupErrors` | boolean | `false`          | If `true`, the plugin will not throw an error if the engine is not reachable at startup.           |
+| `engine.autoUpdate`   | boolean | `true`           | If `true`, the plugin will automatically update the engine collections when the plugin is started. |
+| `adminIndex`          | string  | `device-manager` | The index name where the plugin stores its configuration and devices.                              |
+
+## Collections
+
+In order to customize the collections used by the plugin, you can provide a configuration object for each collection. All of these support the following properties:
+
+- `name`: The name of the collection.
+- `mappings`: The mappings of the collection (according to the [Elasticsearch mapping specification](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html)).
+- `settings`: The settings of the collection (according to the [Elasticsearch settings specification](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html)). This property is optional but can be useful to define the number of shards and replicas of the Elasticsearch index or use custom analyzers.
+
+### Admin collections 
+
+The following collections are used by the plugin. They're automatically created when the plugin is started and shared accross all the engines.
+
+- `devices`: The collection that stores the devices.
+- `config`: The collection that stores the plugin configuration.
+- `payloads`: The collection that stores the raw payloads before they are processed and decoded by the plugin.
+
+### Engine collections
+
+The following collections are used by the plugin. They're automatically created when the plugin is started and are specific to each engine:
+
+- `config`: The collection that stores the engine configuration.
+- `device`: The collection that stores the devices.
+- `measures`: The collection that stores the measures once extracted by the decoders.
+- `asset`: The collection that stores the assets.
+- `assetGroups`: The collection that stores the assets groups.
+- `assetHistory`: The collection that stores the assets history.
+
+# Example
+
+```js
+{
+  "plugins": {
+    "device-manager": {
+      "ignoreStartupErrors": false,
+      "engine": {
+        "autoUpdate": true
+      },
+      "adminIndex": "device-manager",
+      "adminCollections": {
+        "devices": {
+          "name": "devices",
+          "mappings": {
+            "properties": {
+              "name": { "type": "keyword" },
+              "description": { "type": "text" },
+              "type": { "type": "keyword" },
+              "status": { "type": "keyword" },
+              "lastSeen": { "type": "date" }
+            }
+          },
+          "settings": {
+            "index": {
+              "number_of_shards": 1,
+              "number_of_replicas": 1
+            }
+          }
+        },
+        "config": {
+          "name": "config",
+          "mappings": {
+            "properties": {
+              "name": { "type": "keyword" },
+              "value": { "type": "text" }
+            }
+          },
+          "settings": {
+            "index": {
+              "number_of_shards": 1,
+              "number_of_replicas": 1
+            }
+          }
+        },
+        "payloads": {
+          "name": "payloads",
+          "mappings": {
+            "properties": {
+              "payload": { "type": "text" }
+            }
+          }
+        }
+      },
+      "engineCollections": {
+        "config": {
+          "name": "config",
+          "mappings": {
+            "properties": {
+              "name": { "type": "keyword" },
+              "value": { "type": "text" }
+            }
+          }
+        },
+        "devices": {
+          "name": "devices",
+          "mappings": {
+            "properties": {
+              "name": { "type": "keyword" },
+              "description": { "type": "text" },
+              "type": { "type": "keyword" },
+              "status": { "type": "keyword" },
+              "lastSeen": { "type": "date" }
+            }
+          }
+        },
+        "measures": {
+          "name": "measures",
+          "mappings": {
+            "properties": {
+              "deviceId": { "type": "keyword" },
+              "timestamp": { "type": "date" },
+              "measures": { "type": "object" }
+            }
+          }
+        },
+        "asset": {
+          "name": "assets",
+          "mappings": {
+            "properties": {
+              "name": { "type": "keyword" },
+              "description": { 
+                "type": "text",
+                "analyzer": "description" 
+              },
+              "type": { "type": "keyword" },
+              "status": { "type": "keyword" }
+            }
+          },
+          "settings": {
+            // Custom analyzer definition
+            "analysis": {
+              "analyzer": {
+                "description": {
+                  "type": "custom",
+                  "tokenizer": "whitespace"
+                }
+              }
+            }
+          }
+        },
+        "assetGroups": {
+          "name": "assetGroups",
+          "mappings": {
+            "properties": {
+              "name": { "type": "keyword" },
+              "description": { "type": "text" },
+              "type": { "type": "keyword" },
+              "status": { "type": "keyword" }
+            }
+          }
+        },
+        "assetHistory": {
+          "name": "assetHistory",
+          "mappings": {
+            "properties": {
+              "assetId": { "type": "keyword" },
+              "timestamp": { "type": "date" },
+              "status": { "type": "keyword" }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```

docker-compose.yml

@@ -30,7 +30,7 @@ services:
       - esdata01:/usr/share/elasticsearch/data
 
   kuzzle:
-    image: kuzzleio/kuzzle-runner:18
+    image: kuzzleio/kuzzle-runner:20
     restart: unless-stopped
     command: "bash /docker/start-backend.sh"
     profiles: ["backend"]

lib/modules/plugin/DeviceManagerEngine.ts

@@ -320,12 +320,12 @@ export class DeviceManagerEngine extends AbstractEngine<DeviceManagerPlugin> {
       "asset",
       engineGroup,
     );
+    const settings = this.config.engineCollections.asset.settings;
 
-    await this.tryCreateCollection(
-      engineId,
-      InternalCollection.ASSETS,
+    await this.tryCreateCollection(engineId, InternalCollection.ASSETS, {
       mappings,
-    );
+      settings,
+    });
 
     return InternalCollection.ASSETS;
   }
@@ -349,10 +349,15 @@ export class DeviceManagerEngine extends AbstractEngine<DeviceManagerPlugin> {
 
     _.merge(mappings.properties.asset, assetsMappings);
 
+    const settings = this.config.engineCollections.assetHistory.settings;
+
     await this.tryCreateCollection(
       engineId,
       InternalCollection.ASSETS_HISTORY,
-      mappings,
+      {
+        mappings,
+        settings,
+      },
     );
 
     return InternalCollection.ASSETS_HISTORY;
@@ -367,8 +372,11 @@ export class DeviceManagerEngine extends AbstractEngine<DeviceManagerPlugin> {
    * @throws If it failed during the assets groups collection creation
    */
   async createAssetsGroupsCollection(engineId: string) {
+    const settings = this.config.engineCollections.assetGroups.settings;
+
     await this.tryCreateCollection(engineId, InternalCollection.ASSETS_GROUPS, {
       mappings: assetGroupsMappings,
+      settings,
     });
 
     return InternalCollection.ASSETS_GROUPS;
@@ -385,12 +393,12 @@ export class DeviceManagerEngine extends AbstractEngine<DeviceManagerPlugin> {
   async createDevicesCollection(engineId: string) {
     const mappings =
       await this.getDigitalTwinMappingsFromDB<DeviceModelContent>("device");
+    const settings = this.config.engineCollections.device.settings;
 
-    await this.tryCreateCollection(
-      engineId,
-      InternalCollection.DEVICES,
+    await this.tryCreateCollection(engineId, InternalCollection.DEVICES, {
       mappings,
-    );
+      settings,
+    });
 
     return InternalCollection.DEVICES;
   }
@@ -405,12 +413,12 @@ export class DeviceManagerEngine extends AbstractEngine<DeviceManagerPlugin> {
    */
   async createMeasuresCollection(engineId: string, engineGroup: string) {
     const mappings = await this.getMeasuresMappingsFromDB(engineGroup);
+    const settings = this.config.engineCollections.measures.settings;
 
-    await this.tryCreateCollection(
-      engineId,
-      InternalCollection.MEASURES,
+    await this.tryCreateCollection(engineId, InternalCollection.MEASURES, {
       mappings,
-    );
+      settings,
+    });
 
     return InternalCollection.MEASURES;
   }
@@ -427,15 +435,15 @@ export class DeviceManagerEngine extends AbstractEngine<DeviceManagerPlugin> {
   private async tryCreateCollection(
     engineIndex: string,
     collection: string,
-    mappings:
+    options:
       | CollectionMappings
       | {
           mappings?: CollectionMappings;
           settings?: JSONObject;
         },
   ) {
     try {
-      await this.sdk.collection.create(engineIndex, collection, mappings);
+      await this.sdk.collection.create(engineIndex, collection, options);
     } catch (error) {
       throw new InternalError(
         `Failed to create the collection [${collection}] for engine [${engineIndex}]: ${error.message}`,

lib/modules/plugin/DeviceManagerPlugin.ts

@@ -262,17 +262,14 @@ export class DeviceManagerPlugin extends Plugin {
             dynamic: "strict",
             properties: {},
           },
-          settings: {},
         },
         devices: {
           name: "devices",
           mappings: devicesMappings,
-          settings: {},
         },
         payloads: {
           name: "payloads",
           mappings: payloadsMappings,
-          settings: {},
         },
       },
       engineCollections: {
@@ -282,16 +279,24 @@ export class DeviceManagerPlugin extends Plugin {
             dynamic: "strict",
             properties: {},
           },
-          settings: {},
         },
         asset: {
           name: InternalCollection.ASSETS,
           mappings: assetsMappings,
         },
+        assetGroups: {
+          name: InternalCollection.ASSETS_GROUPS,
+        },
+        assetHistory: {
+          name: InternalCollection.ASSETS_HISTORY,
+        },
         device: {
           name: InternalCollection.DEVICES,
           mappings: devicesMappings,
         },
+        measures: {
+          name: InternalCollection.MEASURES,
+        },
       },
     };
     /* eslint-enable sort-keys */
@@ -443,7 +448,10 @@ export class DeviceManagerPlugin extends Plugin {
         });
 
       await this.sdk.collection
-        .create(this.config.adminIndex, "payloads", this.getPayloadsMappings())
+        .create(this.config.adminIndex, "payloads", {
+          mappings: this.getPayloadsMappings(),
+          settings: this.config.adminCollections.payloads.settings,
+        })
         .catch((error) => {
           throw keepStack(
             error,