feat: Extract HDR metadata from HLS manifests (#3116)

In our isTypeSupported polyfill for Cast, we use the HEVC profile ("hevc1.2") as
an indication of the HDR transfer function, which is not always accurate.
The polyfill should stop assuming that 10-bit color HEVC means HDR, and remove
the extra eotf="smpte2084" parameter. Instead, the manifest parser should
extract HDR profile information from the DASH manifest and pass that info along
through the Stream object.

Issue #3116 .

externs/shaka/manifest.js

@@ -225,6 +225,7 @@ shaka.extern.CreateSegmentIndexFunction;
  *   codecs: string,
  *   frameRate: (number|undefined),
  *   pixelAspectRatio: (string|undefined),
+ *   hdr: (string|undefined),
  *   bandwidth: (number|undefined),
  *   width: (number|undefined),
  *   height: (number|undefined),
@@ -276,6 +277,9 @@ shaka.extern.CreateSegmentIndexFunction;
  * @property {(string|undefined)} pixelAspectRatio
  *   <i>Video streams only.</i> <br>
  *   The Stream's pixel aspect ratio
+ * @property {(string|undefined)} hdr
+ *   <i>Video streams only.</i> <br>
+ *   The Stream's HDR info
  * @property {(number|undefined)} bandwidth
  *   <i>Audio and video streams only.</i> <br>
  *   The stream's required bandwidth in bits per second.

externs/shaka/offline.js

@@ -105,6 +105,7 @@ shaka.extern.ManifestDB;
  *   codecs: string,
  *   frameRate: (number|undefined),
  *   pixelAspectRatio: (string|undefined),
+ *   hdr: (string|undefined),
  *   kind: (string|undefined),
  *   language: string,
  *   label: ?string,
@@ -138,6 +139,8 @@ shaka.extern.ManifestDB;
  *   The Stream's framerate in frames per second.
  * @property {(string|undefined)} pixelAspectRatio
  *   The Stream's pixel aspect ratio
+ * @property {(string|undefined)} hdr
+ *   The Stream's HDR info
  * @property {(string|undefined)} kind
  *   The kind of text stream; undefined for audio/video.
  * @property {string} language

externs/shaka/player.js

@@ -206,6 +206,7 @@ shaka.extern.BufferedInfo;
  *   height: ?number,
  *   frameRate: ?number,
  *   pixelAspectRatio: ?string,
+ *   hdr: ?string,
  *   mimeType: ?string,
  *   codecs: ?string,
  *   audioCodec: ?string,
@@ -258,6 +259,8 @@ shaka.extern.BufferedInfo;
  *   The video framerate provided in the manifest, if present.
  * @property {?string} pixelAspectRatio
  *   The video pixel aspect ratio provided in the manifest, if present.
+ * @property {?string} hdr
+ *   The video HDR provided in the manifest, if present.
  * @property {?string} mimeType
  *   The MIME type of the content provided in the manifest.
  * @property {?string} codecs

lib/hls/hls_parser.js

@@ -699,6 +699,8 @@ shaka.hls.HlsParser = class {
       const resolution = tag.getAttributeValue('RESOLUTION');
       const [width, height] = resolution ? resolution.split('x') : [null, null];
 
+      const videoRange = tag.getAttributeValue('VIDEO-RANGE');
+
       const streamInfos = await this.createStreamInfosForVariantTag_(tag,
           resolution, frameRate);
 
@@ -712,7 +714,8 @@ shaka.hls.HlsParser = class {
             bandwidth,
             width,
             height,
-            frameRate);
+            frameRate,
+            videoRange);
       }
       // We do not support AES-128 encryption with HLS yet. If the streamInfos
       // is null because of AES-128 encryption, do not create variants for that.
@@ -969,15 +972,18 @@ shaka.hls.HlsParser = class {
    * @param {?string} width
    * @param {?string} height
    * @param {?string} frameRate
+   * @param {?string} videoRange
    * @return {!Array.<!shaka.extern.Variant>}
    * @private
    */
-  createVariants_(audioInfos, videoInfos, bandwidth, width, height, frameRate) {
+  createVariants_(
+      audioInfos, videoInfos, bandwidth, width, height, frameRate, videoRange) {
     const ContentType = shaka.util.ManifestParserUtils.ContentType;
     const DrmEngine = shaka.media.DrmEngine;
 
     for (const info of videoInfos) {
-      this.addVideoAttributes_(info.stream, width, height, frameRate);
+      this.addVideoAttributes_(
+          info.stream, width, height, frameRate, videoRange);
     }
 
     // In case of audio-only or video-only content or the audio/video is
@@ -2440,13 +2446,15 @@ shaka.hls.HlsParser = class {
    * @param {?string} width
    * @param {?string} height
    * @param {?string} frameRate
+   * @param {?string} videoRange
    * @private
    */
-  addVideoAttributes_(stream, width, height, frameRate) {
+  addVideoAttributes_(stream, width, height, frameRate, videoRange) {
     if (stream) {
       stream.width = Number(width) || undefined;
       stream.height = Number(height) || undefined;
       stream.frameRate = Number(frameRate) || undefined;
+      stream.hdr = videoRange || undefined;
     }
   }
 

lib/offline/indexeddb/v1_storage_cell.js

@@ -155,6 +155,7 @@ shaka.offline.indexeddb.V1StorageCell = class
       codecs: old.codecs,
       frameRate: old.frameRate,
       pixelAspectRatio: undefined,
+      hdr: undefined,
       kind: old.kind,
       language: old.language,
       label: old.label,

lib/offline/indexeddb/v2_storage_cell.js

@@ -104,6 +104,7 @@ shaka.offline.indexeddb.V2StorageCell = class
       codecs: old.codecs,
       frameRate: old.frameRate,
       pixelAspectRatio: old.pixelAspectRatio,
+      hdr: undefined,
       kind: old.kind,
       language: old.language,
       label: old.label,

lib/offline/manifest_converter.js

@@ -177,6 +177,7 @@ shaka.offline.ManifestConverter = class {
       height: streamDB.height || undefined,
       frameRate: streamDB.frameRate,
       pixelAspectRatio: streamDB.pixelAspectRatio,
+      hdr: streamDB.hdr,
       kind: streamDB.kind,
       encrypted: streamDB.encrypted,
       drmInfos: [],

lib/offline/storage.js

@@ -1106,6 +1106,7 @@ shaka.offline.Storage = class {
       codecs: stream.codecs,
       frameRate: stream.frameRate,
       pixelAspectRatio: stream.pixelAspectRatio,
+      hdr: stream.hdr,
       kind: stream.kind,
       language: stream.language,
       label: stream.label,

lib/util/mime_utils.js

@@ -44,6 +44,9 @@ shaka.util.MimeUtils = class {
         components.push(mimeKey + '="' + value + '"');
       }
     });
+    if (stream.hdr == 'PQ') {
+      components.push('eotf="smpte2084"');
+    }
 
     return components.join(';');
   }

lib/util/stream_utils.js

@@ -359,6 +359,7 @@ shaka.util.StreamUtils = class {
       height: null,
       frameRate: null,
       pixelAspectRatio: null,
+      hdr: null,
       mimeType: mimeType,
       codecs: codecs.join(', '),
       audioCodec: audioCodec,
@@ -422,6 +423,7 @@ shaka.util.StreamUtils = class {
       height: null,
       frameRate: null,
       pixelAspectRatio: null,
+      hdr: null,
       mimeType: stream.mimeType,
       codecs: stream.codecs || null,
       audioCodec: null,
@@ -536,6 +538,7 @@ shaka.util.StreamUtils = class {
       height: null,
       frameRate: null,
       pixelAspectRatio: null,
+      hdr: null,
       mimeType: null,
       codecs: null,
       audioCodec: null,

test/hls/hls_parser_unit.js

@@ -958,6 +958,67 @@ describe('HlsParser', () => {
     await testHlsParser(master, media, manifest);
   });
 
+  it('parses manifest with HDR metadata', async () => {
+    const master = [
+      '#EXTM3U\n',
+      '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud1",LANGUAGE="eng",',
+      'URI="audio"\n',
+      '#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="sub1",LANGUAGE="eng",',
+      'FORCED=YES,URI="text"\n',
+      '#EXT-X-STREAM-INF:BANDWIDTH=200,CODECS="avc1,mp4a",VIDEO-RANGE=PQ,',
+      'RESOLUTION=960x540,FRAME-RATE=60,AUDIO="aud1",SUBTITLES="sub1"\n',
+      'video\n',
+    ].join('');
+
+    const media = [
+      '#EXTM3U\n',
+      '#EXT-X-PLAYLIST-TYPE:VOD\n',
+      '#EXT-X-MAP:URI="init.mp4",BYTERANGE="616@0"\n',
+      '#EXTINF:5,\n',
+      '#EXT-X-BYTERANGE:121090@616\n',
+      'main.mp4',
+    ].join('');
+
+    const textMedia = [
+      '#EXTM3U\n',
+      '#EXT-X-PLAYLIST-TYPE:VOD\n',
+      '#EXTINF:5,\n',
+      '#EXT-X-BYTERANGE:121090@616\n',
+      'main.vtt',
+    ].join('');
+
+    const manifest = shaka.test.ManifestGenerator.generate((manifest) => {
+      manifest.anyTimeline();
+      manifest.addPartialVariant((variant) => {
+        variant.addPartialStream(ContentType.VIDEO, (stream) => {
+          stream.mime('video/mp4', 'avc1');
+          stream.hdr = 'PQ';
+        });
+        variant.addPartialStream(ContentType.AUDIO, (stream) => {
+          stream.mime('audio/mp4', 'mp4a');
+        });
+      });
+      manifest.addPartialTextStream((stream) => {
+        stream.language = 'en';
+        stream.forced = true;
+        stream.kind = TextStreamKind.SUBTITLE;
+        stream.mime('text/vtt', '');
+      });
+    });
+
+    fakeNetEngine
+        .setResponseText('test:/master', master)
+        .setResponseText('test:/audio', media)
+        .setResponseText('test:/video', media)
+        .setResponseText('test:/text', textMedia)
+        .setResponseText('test:/main.vtt', vttText)
+        .setResponseValue('test:/init.mp4', initSegmentData)
+        .setResponseValue('test:/main.mp4', segmentData);
+
+    const actual = await parser.start('test:/master', playerInterface);
+    expect(actual).toEqual(manifest);
+  });
+
   it('parses manifest with SUBTITLES', async () => {
     const master = [
       '#EXTM3U\n',

test/offline/manifest_convert_unit.js

@@ -335,6 +335,7 @@ describe('ManifestConverter', () => {
       codecs: 'avc1.42c01e',
       frameRate: 22,
       pixelAspectRatio: '59:54',
+      hdr: undefined,
       kind: undefined,
       language: '',
       label: null,
@@ -381,6 +382,7 @@ describe('ManifestConverter', () => {
       codecs: 'mp4a.40.2',
       frameRate: undefined,
       pixelAspectRatio: undefined,
+      hdr: undefined,
       kind: undefined,
       language: 'en',
       label: null,
@@ -426,6 +428,7 @@ describe('ManifestConverter', () => {
       codecs: '',
       frameRate: undefined,
       pixelAspectRatio: undefined,
+      hdr: undefined,
       kind: undefined,
       language: 'en',
       label: null,
@@ -478,6 +481,7 @@ describe('ManifestConverter', () => {
       codecs: streamDb.codecs,
       frameRate: streamDb.frameRate,
       pixelAspectRatio: streamDb.pixelAspectRatio,
+      hdr: streamDb.hdr,
       width: streamDb.width || undefined,
       height: streamDb.height || undefined,
       kind: streamDb.kind,

test/offline/storage_integration.js

@@ -1295,6 +1295,7 @@ filterDescribe('Storage', storageSupport, () => {
       height: height,
       frameRate: 30,
       pixelAspectRatio: '59:54',
+      hdr: null,
       mimeType: 'video/mp4,audio/mp4',
       codecs: 'mp4,mp4',
       audioCodec: 'mp4',
@@ -1333,6 +1334,7 @@ filterDescribe('Storage', storageSupport, () => {
       height: null,
       frameRate: null,
       pixelAspectRatio: null,
+      hdr: null,
       mimeType: 'text/vtt',
       codecs: 'vtt',
       audioCodec: null,

test/player_unit.js

@@ -1059,6 +1059,7 @@ describe('Player', () => {
           height: 200,
           frameRate: 1000000 / 42000,
           pixelAspectRatio: '59:54',
+          hdr: null,
           mimeType: 'video/mp4',
           codecs: 'avc1.4d401f, mp4a.40.2',
           audioCodec: 'mp4a.40.2',
@@ -1089,6 +1090,7 @@ describe('Player', () => {
           height: 400,
           frameRate: 24,
           pixelAspectRatio: '59:54',
+          hdr: null,
           mimeType: 'video/mp4',
           codecs: 'avc1.4d401f, mp4a.40.2',
           audioCodec: 'mp4a.40.2',
@@ -1119,6 +1121,7 @@ describe('Player', () => {
           height: 200,
           frameRate: 1000000 / 42000,
           pixelAspectRatio: '59:54',
+          hdr: null,
           mimeType: 'video/mp4',
           codecs: 'avc1.4d401f, mp4a.40.2',
           audioCodec: 'mp4a.40.2',
@@ -1149,6 +1152,7 @@ describe('Player', () => {
           height: 400,
           frameRate: 24,
           pixelAspectRatio: '59:54',
+          hdr: null,
           mimeType: 'video/mp4',
           codecs: 'avc1.4d401f, mp4a.40.2',
           audioCodec: 'mp4a.40.2',
@@ -1179,6 +1183,7 @@ describe('Player', () => {
           height: 200,
           frameRate: 1000000 / 42000,
           pixelAspectRatio: '59:54',
+          hdr: null,
           mimeType: 'video/mp4',
           codecs: 'avc1.4d401f, mp4a.40.2',
           audioCodec: 'mp4a.40.2',
@@ -1209,6 +1214,7 @@ describe('Player', () => {
           height: 400,
           frameRate: 24,
           pixelAspectRatio: '59:54',
+          hdr: null,
           mimeType: 'video/mp4',
           codecs: 'avc1.4d401f, mp4a.40.2',
           audioCodec: 'mp4a.40.2',
@@ -1239,6 +1245,7 @@ describe('Player', () => {
           height: 200,
           frameRate: 1000000 / 42000,
           pixelAspectRatio: '59:54',
+          hdr: null,
           mimeType: 'video/mp4',
           codecs: 'avc1.4d401f, mp4a.40.2',
           audioCodec: 'mp4a.40.2',
@@ -1269,6 +1276,7 @@ describe('Player', () => {
           height: 400,
           frameRate: 24,
           pixelAspectRatio: '59:54',
+          hdr: null,
           mimeType: 'video/mp4',
           codecs: 'avc1.4d401f, mp4a.40.2',
           audioCodec: 'mp4a.40.2',
@@ -1314,6 +1322,7 @@ describe('Player', () => {
           height: null,
           frameRate: null,
           pixelAspectRatio: null,
+          hdr: null,
           videoId: null,
           audioId: null,
           originalAudioId: null,
@@ -1344,6 +1353,7 @@ describe('Player', () => {
           height: null,
           frameRate: null,
           pixelAspectRatio: null,
+          hdr: null,
           videoId: null,
           audioId: null,
           originalAudioId: null,
@@ -1374,6 +1384,7 @@ describe('Player', () => {
           height: null,
           frameRate: null,
           pixelAspectRatio: null,
+          hdr: null,
           videoId: null,
           audioId: null,
           originalAudioId: null,