feat(server): qsv hardware decoding and tone-mapping (immich-app#9689)

* qsv hw decoding and tone-mapping * fix vaapi * add tests * formatting * handle device name without path
magicedy · May 23, 2024 · a5e8b45 · a5e8b45
1 parent 13cbdf6
commit a5e8b45
Show file tree

Hide file tree

Showing 3 changed files with 132 additions and 8 deletions.
diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts
@@ -1482,6 +1482,74 @@ describe(MediaService.name, () => {
       expect(mediaMock.transcode).not.toHaveBeenCalled();
     });
 
+    it('should use hardware decoding for qsv if enabled', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      systemMock.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHWAccel.QSV, accelDecode: true },
+      });
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: expect.arrayContaining(['-hwaccel qsv', '-async_depth 4', '-threads 1']),
+          outputOptions: expect.arrayContaining([
+            expect.stringContaining('scale_qsv=-1:720:async_depth=4:mode=hq:format=nv12'),
+          ]),
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should use hardware tone-mapping for qsv if hardware decoding is enabled and should tone map', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
+      systemMock.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHWAccel.QSV, accelDecode: true },
+      });
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: expect.arrayContaining(['-hwaccel qsv', '-async_depth 4', '-threads 1']),
+          outputOptions: expect.arrayContaining([
+            expect.stringContaining(
+              'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:transfer=bt709,hwmap=derive_device=vaapi:reverse=1',
+            ),
+          ]),
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should use preferred device for qsv when hardware decoding', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128', 'renderD129', 'renderD130']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      systemMock.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHWAccel.QSV, accelDecode: true, preferredHwDevice: 'renderD129' },
+      });
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: expect.arrayContaining(['-hwaccel qsv', '-qsv_device /dev/dri/renderD129']),
+          outputOptions: expect.any(Array),
+          twoPass: false,
+        },
+      );
+    });
+
     it('should set options for vaapi', async () => {
       storageMock.readdir.mockResolvedValue(['renderD128']);
       mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);

diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts
@@ -38,7 +38,8 @@ import {
   HEVCConfig,
   NvencHwDecodeConfig,
   NvencSwDecodeConfig,
-  QSVConfig,
+  QsvHwDecodeConfig,
+  QsvSwDecodeConfig,
   RkmppHwDecodeConfig,
   RkmppSwDecodeConfig,
   ThumbnailConfig,
@@ -499,7 +500,9 @@ export class MediaService {
         break;
       }
       case TranscodeHWAccel.QSV: {
-        handler = new QSVConfig(config, await this.getDevices());
+        handler = config.accelDecode
+          ? new QsvHwDecodeConfig(config, await this.getDevices())
+          : new QsvSwDecodeConfig(config, await this.getDevices());
         break;
       }
       case TranscodeHWAccel.VAAPI: {

diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts
@@ -302,18 +302,18 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
     return this.config.gopSize;
   }
 
-  getPreferredHardwareDevice(): string | null {
+  getPreferredHardwareDevice(): string | undefined {
     const device = this.config.preferredHwDevice;
     if (device === 'auto') {
-      return null;
+      return;
     }
 
     const deviceName = device.replace('/dev/dri/', '');
     if (!this.devices.includes(deviceName)) {
       throw new Error(`Device '${device}' does not exist`);
     }
 
-    return device;
+    return `/dev/dri/${deviceName}`;
   }
 }
 
@@ -567,15 +567,15 @@ export class NvencHwDecodeConfig extends NvencSwDecodeConfig {
   }
 }
 
-export class QSVConfig extends BaseHWConfig {
+export class QsvSwDecodeConfig extends BaseHWConfig {
   getBaseInputOptions() {
     if (this.devices.length === 0) {
       throw new Error('No QSV device found');
     }
 
     let qsvString = '';
     const hwDevice = this.getPreferredHardwareDevice();
-    if (hwDevice !== null) {
+    if (hwDevice) {
       qsvString = `,child_device=${hwDevice}`;
     }
 
@@ -643,14 +643,67 @@ export class QSVConfig extends BaseHWConfig {
   }
 }
 
+export class QsvHwDecodeConfig extends QsvSwDecodeConfig {
+  getBaseInputOptions() {
+    if (this.devices.length === 0) {
+      throw new Error('No QSV device found');
+    }
+
+    const options = ['-hwaccel qsv', '-async_depth 4', '-threads 1'];
+    const hwDevice = this.getPreferredHardwareDevice();
+    if (hwDevice) {
+      options.push(`-qsv_device ${hwDevice}`);
+    }
+
+    return options;
+  }
+
+  getFilterOptions(videoStream: VideoStreamInfo) {
+    const options = [];
+    if (this.shouldScale(videoStream) || !this.shouldToneMap(videoStream)) {
+      let scaling = `scale_qsv=${this.getScaling(videoStream)}:async_depth=4:mode=hq`;
+      if (!this.shouldToneMap(videoStream)) {
+        scaling += ':format=nv12';
+      }
+      options.push(scaling);
+    }
+
+    options.push(...this.getToneMapping(videoStream));
+    return options;
+  }
+
+  getToneMapping(videoStream: VideoStreamInfo): string[] {
+    if (!this.shouldToneMap(videoStream)) {
+      return [];
+    }
+
+    const colors = this.getColors();
+    const tonemapOptions = [
+      'desat=0',
+      'format=nv12',
+      `matrix=${colors.matrix}`,
+      `primaries=${colors.primaries}`,
+      'range=pc',
+      `tonemap=${this.config.tonemap}`,
+      `transfer=${colors.transfer}`,
+    ];
+
+    return [
+      'hwmap=derive_device=opencl',
+      `tonemap_opencl=${tonemapOptions.join(':')}`,
+      'hwmap=derive_device=vaapi:reverse=1',
+    ];
+  }
+}
+
 export class VAAPIConfig extends BaseHWConfig {
   getBaseInputOptions() {
     if (this.devices.length === 0) {
       throw new Error('No VAAPI device found');
     }
 
     let hwDevice = this.getPreferredHardwareDevice();
-    if (hwDevice === null) {
+    if (!hwDevice) {
       hwDevice = `/dev/dri/${this.devices[0]}`;
     }