fix: Improve device icon serving (#25299)

data/configuration.example.yaml

@@ -1,5 +1,5 @@
 # Indicates the configuration version (used by configuration migrations)
-version: 2
+version: 4
 
 # Home Assistant integration (MQTT discovery)
 homeassistant:

lib/extension/bridge.ts

@@ -432,6 +432,16 @@ export default class Bridge extends Extension {
         const ID = message.id;
         const entity = this.getEntity(entityType, ID);
         const oldOptions = objectAssignDeep({}, cleanup(entity.options));
+
+        if (message.options.icon) {
+            const base64Match = utils.matchBase64File(message.options.icon);
+            if (base64Match) {
+                const fileSettings = utils.saveBase64DeviceIcon(base64Match);
+                message.options.icon = fileSettings;
+                logger.debug(`Saved base64 image as file to '${fileSettings}'`);
+            }
+        }
+
         const restartRequired = settings.changeEntityOptions(ID, message.options);
         if (restartRequired) this.restartRequired = true;
         const newOptions = cleanup(entity.options);

lib/extension/frontend.ts

@@ -18,6 +18,7 @@ import WebSocket from 'ws';
 
 import frontend from 'zigbee2mqtt-frontend';
 
+import data from '../util/data';
 import logger from '../util/logger';
 import * as settings from '../util/settings';
 import utils from '../util/utils';
@@ -35,6 +36,7 @@ export default class Frontend extends Extension {
     private authToken: string | undefined;
     private server!: Server;
     private fileServer!: RequestHandler;
+    private deviceIconsFileServer!: RequestHandler;
     private wss!: WebSocket.Server;
     private baseUrl: string;
 
@@ -89,6 +91,7 @@ export default class Frontend extends Extension {
             },
         };
         this.fileServer = expressStaticGzip(frontend.getPath(), options);
+        this.deviceIconsFileServer = expressStaticGzip(data.joinPath('device_icons'), options);
         this.wss = new WebSocket.Server({noServer: true, path: posix.join(this.baseUrl, 'api')});
 
         this.wss.on('connection', this.onWebSocketConnection);
@@ -144,7 +147,13 @@ export default class Frontend extends Extension {
         request.url = '/' + newUrl;
         request.path = request.url;
 
-        this.fileServer(request, response, fin);
+        if (newUrl.startsWith('device_icons/')) {
+            request.path = request.path.replace('device_icons/', '');
+            request.url = request.url.replace('/device_icons', '');
+            this.deviceIconsFileServer(request, response, fin);
+        } else {
+            this.fileServer(request, response, fin);
+        }
     }
 
     private authenticate(request: IncomingMessage, cb: (authenticate: boolean) => void): void {

lib/util/settings.schema.json

@@ -804,7 +804,7 @@
                 "icon": {
                     "type": "string",
                     "title": "Icon",
-                    "description": "The user-defined device icon for the frontend. It can be a full URL link to an image (e.g. https://SOME.SITE/MODEL123.jpg) (you cannot use a path to a local file) or base64 encoded data URL (e.g. image/svg+xml;base64,PHN2ZyB3aW....R0aD)"
+                    "description": "The user-defined device icon for the frontend. It can be a full URL link to an image (e.g. https://SOME.SITE/MODEL123.jpg) or a path to a local file inside the `device_icons` directory."
                 },
                 "homeassistant": {
                     "type": ["object", "null"],

lib/util/settings.ts

@@ -9,7 +9,10 @@ import utils from './utils';
 import yaml, {YAMLFileException} from './yaml';
 
 export {schemaJson};
-export const CURRENT_VERSION = 3;
+// When updating also update:
+// - https://github.com/Koenkk/zigbee2mqtt/blob/dev/data/configuration.example.yaml#L2
+// - https://github.com/zigbee2mqtt/hassio-zigbee2mqtt/blob/master/common/rootfs/docker-entrypoint.sh#L54
+export const CURRENT_VERSION = 4;
 /** NOTE: by order of priority, lower index is lower level (more important) */
 export const LOG_LEVELS: readonly string[] = ['error', 'warning', 'info', 'debug'] as const;
 export type LogLevel = 'error' | 'warning' | 'info' | 'debug';
@@ -246,6 +249,9 @@ export function validate(): string[] {
         if (names.includes(e.friendly_name)) errors.push(`Duplicate friendly_name '${e.friendly_name}' found`);
         errors.push(...utils.validateFriendlyName(e.friendly_name));
         names.push(e.friendly_name);
+        if ('icon' in e && e.icon && !e.icon.startsWith('http://') && !e.icon.startsWith('https://') && !e.icon.startsWith('device_icons/')) {
+            errors.push(`Device icon of '${e.friendly_name}' should start with 'device_icons/', got '${e.icon}'`);
+        }
         if (e.qos != null && ![0, 1, 2].includes(e.qos)) {
             errors.push(`QOS for '${e.friendly_name}' not valid, should be 0, 1 or 2 got ${e.qos}`);
         }

lib/util/settingsMigration.ts

@@ -2,6 +2,7 @@ import {copyFileSync, writeFileSync} from 'node:fs';
 
 import data from './data';
 import * as settings from './settings';
+import utils from './utils';
 
 interface SettingsMigration {
     path: string[];
@@ -28,7 +29,7 @@ interface SettingsCustomHandler extends Omit<SettingsMigration, 'path'> {
     execute: (currentSettings: Partial<Settings>) => [validPath: boolean, previousValue: unknown, changed: boolean];
 }
 
-const SUPPORTED_VERSIONS: Settings['version'][] = [undefined, 2, settings.CURRENT_VERSION];
+const SUPPORTED_VERSIONS: Settings['version'][] = [undefined, 2, 3, settings.CURRENT_VERSION];
 
 function backupSettings(version: number): void {
     const filePath = data.joinPath('configuration.yaml');
@@ -438,6 +439,43 @@ function migrateToThree(
     );
 }
 
+function migrateToFour(
+    currentSettings: Partial<Settings>,
+    transfers: SettingsTransfer[],
+    changes: SettingsChange[],
+    additions: SettingsAdd[],
+    removals: SettingsRemove[],
+    customHandlers: SettingsCustomHandler[],
+): void {
+    transfers.push();
+    changes.push({
+        path: ['version'],
+        note: `Migrated settings to version 4`,
+        newValue: 4,
+    });
+    additions.push();
+    removals.push();
+
+    const saveBase64DeviceIconsAsImage = (currentSettings: Partial<Settings>): ReturnType<SettingsCustomHandler['execute']> => {
+        const [validPath, previousValue] = getValue(currentSettings, ['devices']);
+
+        for (const deviceKey in currentSettings.devices) {
+            const base64Match = utils.matchBase64File(currentSettings.devices[deviceKey].icon ?? '');
+            if (base64Match) {
+                currentSettings.devices[deviceKey].icon = utils.saveBase64DeviceIcon(base64Match);
+            }
+        }
+
+        return [validPath, previousValue, validPath];
+    };
+
+    customHandlers.push({
+        note: `Device icons are now saved as images.`,
+        noteIf: () => true,
+        execute: (currentSettings) => saveBase64DeviceIconsAsImage(currentSettings),
+    });
+}
+
 /**
  * Order of execution:
  * - Transfer
@@ -482,7 +520,11 @@ export function migrateIfNecessary(): void {
             migrationNotesFileName = 'migration-2-to-3.log';
 
             migrateToThree(currentSettings, transfers, changes, additions, removals, customHandlers);
-        } /* else if (currentSettings.version === 2.1) {} */
+        } else if (currentSettings.version === 3) {
+            migrationNotesFileName = 'migration-3-to-4.log';
+
+            migrateToFour(currentSettings, transfers, changes, additions, removals, customHandlers);
+        }
 
         for (const transfer of transfers) {
             const [validPath, previousValue, transfered] = transferValue(currentSettings, transfer);

lib/util/utils.ts

@@ -2,12 +2,17 @@ import type {Zigbee2MQTTAPI, Zigbee2MQTTResponse, Zigbee2MQTTResponseEndpoints,
 import type * as zhc from 'zigbee-herdsman-converters';
 
 import assert from 'node:assert';
+import crypto from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
 
 import equals from 'fast-deep-equal/es6';
 import humanizeDuration from 'humanize-duration';
 
+import data from './data';
+
+const BASE64_IMAGE_REGEX = new RegExp(`data:image/(?<extension>.+);base64,(?<data>.+)`);
+
 function pad(num: number): string {
     const norm = Math.floor(Math.abs(num));
     return (norm < 10 ? '0' : '') + norm;
@@ -370,10 +375,30 @@ function deviceNotCoordinator(device: zh.Device): boolean {
     return device.type !== 'Coordinator';
 }
 
+function matchBase64File(value: string): {extension: string; data: string} | false {
+    const match = value.match(BASE64_IMAGE_REGEX);
+    if (match) {
+        assert(match.groups?.extension && match.groups?.data);
+        return {extension: match.groups.extension, data: match.groups.data};
+    }
+    return false;
+}
+
+function saveBase64DeviceIcon(base64Match: {extension: string; data: string}): string {
+    const md5Hash = crypto.createHash('md5').update(base64Match.data).digest('hex');
+    const fileSettings = `device_icons/${md5Hash}.${base64Match.extension}`;
+    const file = path.join(data.getPath(), fileSettings);
+    fs.mkdirSync(path.dirname(file), {recursive: true});
+    fs.writeFileSync(file, base64Match.data, {encoding: 'base64'});
+    return fileSettings;
+}
+
 /* v8 ignore next */
 const noop = (): void => {};
 
 export default {
+    matchBase64File,
+    saveBase64DeviceIcon,
     capitalize,
     getZigbee2MQTTVersion,
     getDependencyVersion,

test/extensions/bridge.test.ts

@@ -40,6 +40,8 @@ const mocksClear = [
     devices.bulb.removeFromNetwork,
 ];
 
+const deviceIconsDir = path.join(data.mockDir, 'device_icons');
+
 describe('Extension: Bridge', () => {
     let controller: Controller;
     let mockRestart: Mock;
@@ -80,6 +82,7 @@ describe('Extension: Bridge', () => {
         extension.restartRequired = false;
         // @ts-expect-error private
         controller.state.state = {[devices.bulb.ieeeAddr]: {brightness: 50}};
+        fs.rmSync(deviceIconsDir, {force: true, recursive: true});
     });
 
     afterAll(async () => {
@@ -3241,6 +3244,38 @@ describe('Extension: Bridge', () => {
         );
     });
 
+    it.each([
+        ['data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAJ', 'device_icons/effcad234beeb56ea7c457cf2d36d10b.png', true],
+        ['some_icon.png', 'some_icon.png', false],
+    ])('Should save as image as file when changing device icon', async (mqttIcon, settingsIcon, checkFileExists) => {
+        mockMQTTPublishAsync.mockClear();
+        mockMQTTEvents.message('zigbee2mqtt/bridge/request/device/options', stringify({options: {icon: mqttIcon}, id: 'bulb'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb')).toStrictEqual({
+            ID: '0x000b57fffec6a5b2',
+            friendly_name: 'bulb',
+            icon: settingsIcon,
+            description: 'this is my bulb',
+            retain: true,
+        });
+        if (checkFileExists) {
+            expect(fs.existsSync(path.join(data.mockDir, settingsIcon))).toBeTruthy();
+        }
+        expect(mockMQTTPublishAsync).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/options',
+            stringify({
+                data: {
+                    from: {retain: true, description: 'this is my bulb'},
+                    to: {retain: true, description: 'this is my bulb', icon: settingsIcon},
+                    id: 'bulb',
+                    restart_required: false,
+                },
+                status: 'ok',
+            }),
+            {retain: false, qos: 0},
+        );
+    });
+
     it('Should allow to remove device option', async () => {
         mockMQTTPublishAsync.mockClear();
         settings.set(['devices', '0x000b57fffec6a5b2', 'qos'], 1);