feat(cea): Add CEA parser for TS (#4697)

Closes #3674

Co-authored-by: Joey Parrish <[email protected]>

README.md

@@ -225,11 +225,9 @@ Shaka Player supports:
   - TTML
     - Supported in both XML form and embedded in MP4
   - CEA-608
-    - Supported embedded in MP4
-    - With help from [mux.js][] v6.2.0+, supported embedded in TS
+    - Supported embedded in MP4 and TS
   - CEA-708
-    - Supported embedded in MP4
-    - With help from [mux.js][] v6.2.0+, supported embedded in TS
+    - Supported embedded in MP4 and TS
   - SubRip (SRT)
     - UTF-8 encoding only
   - LyRiCs (LRC)

build/types/cea

@@ -12,3 +12,4 @@
 +../../lib/cea/i_cea_parser.js
 +../../lib/cea/mp4_cea_parser.js
 +../../lib/cea/sei_processor.js
++../../lib/cea/ts_cea_parser.js

externs/mux.js

@@ -23,29 +23,6 @@ var muxjs = {};
 muxjs.mp4 = {};
 
 
-/** @const */
-muxjs.mp4.probe = class {
-  /**
-   * Parses an MP4 initialization segment and extracts the timescale
-   * values for any declared tracks.
-   *
-   * @param {Uint8Array} init The bytes of the init segment
-   * @return {!Object.<number, number>} a hash of track ids to timescale
-   * values or null if the init segment is malformed.
-   */
-  static timescale(init) {}
-
-  /**
-    * Find the trackIds of the video tracks in this source.
-    * Found by parsing the Handler Reference and Track Header Boxes:
-    *
-    * @param {Uint8Array} init The bytes of the init segment for this source
-    * @return {!Array.<number>} A list of trackIds
-   **/
-  static videoTrackIds(init) {}
-};
-
-
 muxjs.mp4.Transmuxer = class {
   /** @param {Object=} options */
   constructor(options) {}
@@ -74,100 +51,19 @@ muxjs.mp4.Transmuxer = class {
 
   /** Remove all handlers and clean up. */
   dispose() {}
-
-  /** Reset captions. */
-  resetCaptions() {}
 };
 
 
 /**
  * @typedef {{
  *   initSegment: !Uint8Array,
- *   data: !Uint8Array,
- *   captions: !Array
+ *   data: !Uint8Array
  * }}
  *
  * @description Transmuxed data from mux.js.
  * @property {!Uint8Array} initSegment
  * @property {!Uint8Array} data
- * @property {!Array} captions
  * @exportDoc
  */
 muxjs.mp4.Transmuxer.Segment;
 
-
-muxjs.mp4.CaptionParser = class {
-  /**
-   * Parser for CEA closed captions embedded in video streams for Dash.
-   * @constructor
-   * @struct
-   */
-  constructor() {}
-
-  /** Initializes the closed caption parser. */
-  init() {}
-
-  /**
-   * Return true if a new video track is selected or if the timescale is
-   * changed.
-   * @param {!Array.<number>} videoTrackIds A list of video tracks found in the
-   *    init segment.
-   * @param {!Object.<number, number>} timescales The map of track Ids and the
-   *    tracks' timescales in the init segment.
-   * @return {boolean}
-   */
-  isNewInit(videoTrackIds, timescales) {}
-
-  /**
-   * Parses embedded CEA closed captions and interacts with the underlying
-   * CaptionStream, and return the parsed captions.
-   * @param {!Uint8Array} segment The fmp4 segment containing embedded captions
-   * @param {!Array.<number>} videoTrackIds A list of video tracks found in the
-   *    init segment.
-   * @param {!Object.<number, number>} timescales The timescales found in the
-   *    init segment.
-   * @return {muxjs.mp4.ParsedClosedCaptions}
-   */
-  parse(segment, videoTrackIds, timescales) {}
-
-  /** Clear the parsed closed captions data for new data. */
-  clearParsedCaptions() {}
-
-  /** Reset the captions stream. */
-  resetCaptionStream() {}
-};
-
-
-/**
- * @typedef {{
- *   captionStreams: Object.<string, boolean>,
- *   captions: !Array.<muxjs.mp4.ClosedCaption>
- * }}
- *
- * @description closed captions data parsed from mux.js caption parser.
- * @property {Object.<string, boolean>} captionStreams
- * @property {Array.<muxjs.mp4.ClosedCaption>} captions
- */
-muxjs.mp4.ParsedClosedCaptions;
-
-
-/**
- * @typedef {{
- *   startPts: number,
- *   endPts: number,
- *   startTime: number,
- *   endTime: number,
- *   stream: string,
- *   text: string
- * }}
- *
- * @description closed caption parsed from mux.js caption parser.
- * @property {number} startPts
- * @property {number} endPts
- * @property {number} startTime
- * @property {number} endTime
- * @property {string} stream The channel id of the closed caption.
- * @property {string} text The content of the closed caption.
- */
-muxjs.mp4.ClosedCaption;
-

externs/shaka/player.js

@@ -1012,10 +1012,8 @@ shaka.extern.ManifestConfiguration;
  *   the default value unless you have a good reason not to.
  * @property {boolean} forceTransmux
  *   If this is <code>true</code>, we will transmux AAC and TS content even if
- *   not strictly necessary for the assets to be played.  Shaka Player
- *   currently only supports CEA 708 captions by transmuxing, so this value is
- *   necessary for enabling them on platforms with native TS support like Edge
- *   or Chromecast. This value defaults to <code>false</code>.
+ *   not strictly necessary for the assets to be played.
+ *   This value defaults to <code>false</code>.
  * @property {number} safeSeekOffset
  *   The amount of seconds that should be added when repositioning the playhead
  *   after falling out of the availability window or seek. This gives the player

lib/cea/cea608_data_channel.js

@@ -54,7 +54,7 @@ shaka.cea.Cea608DataChannel = class {
      * Points to current buffer.
      * @private {!shaka.cea.Cea608Memory}
      */
-    this.curbuf_ = this.displayedMemory_;
+    this.curbuf_ = this.nonDisplayedMemory_;
 
     /**
      * End time of the previous caption, serves as start time of next caption.
@@ -73,14 +73,25 @@ shaka.cea.Cea608DataChannel = class {
    * Resets channel state.
    */
   reset() {
-    this.type_ = shaka.cea.Cea608DataChannel.CaptionType.PAINTON;
-    this.curbuf_ = this.displayedMemory_;
+    this.type_ = shaka.cea.Cea608DataChannel.CaptionType.NONE;
+    this.curbuf_ = this.nonDisplayedMemory_;
     this.lastcp_ = null;
     this.displayedMemory_.reset();
     this.nonDisplayedMemory_.reset();
     this.text_.reset();
   }
 
+  /**
+   * Set the initial PTS, which may not be 0 if we start decoding at a later
+   * point in the stream.  Without this, the first cue's startTime can be way
+   * off.
+   *
+   * @param {number} pts
+   */
+  firstPts(pts) {
+    this.prevEndTime_ = pts;
+  }
+
   /**
    * Gets the row index from a Preamble Address Code byte pair.
    * @param {number} b1 Byte 1.
@@ -155,12 +166,12 @@ shaka.cea.Cea608DataChannel = class {
     }
     buf.setRow(row);
 
-    this.curbuf_.setUnderline(underline);
-    this.curbuf_.setItalics(italics);
-    this.curbuf_.setTextColor(textColor);
+    buf.setUnderline(underline);
+    buf.setItalics(italics);
+    buf.setTextColor(textColor);
 
     // Clear the background color, since new row (PAC) should reset ALL styles.
-    this.curbuf_.setBackgroundColor(shaka.cea.CeaUtils.DEFAULT_BG_COLOR);
+    buf.setBackgroundColor(shaka.cea.CeaUtils.DEFAULT_BG_COLOR);
   }
 
   /**

lib/cea/cea_decoder.js

@@ -77,6 +77,11 @@ shaka.cea.CeaDecoder = class {
      */
     this.serviceNumberToService_ = new Map();
 
+    /**
+     * @private {boolean}
+     */
+    this.waitingForFirstPacket_ = true;
+
     this.reset();
   }
 
@@ -106,6 +111,7 @@ shaka.cea.CeaDecoder = class {
     for (const stream of this.cea608ModeToStream_.values()) {
       stream.reset();
     }
+    this.waitingForFirstPacket_ = true;
   }
 
   /**
@@ -114,6 +120,13 @@ shaka.cea.CeaDecoder = class {
    * @override
    */
   extract(userDataSeiMessage, pts) {
+    if (this.waitingForFirstPacket_) {
+      for (const stream of this.cea608ModeToStream_.values()) {
+        stream.firstPts(pts);
+      }
+      this.waitingForFirstPacket_ = false;
+    }
+
     const reader = new shaka.util.DataViewReader(
         userDataSeiMessage, shaka.util.DataViewReader.Endianness.BIG_ENDIAN);
 

lib/cea/ts_cea_parser.js

@@ -0,0 +1,64 @@
+/*! @license
+ * Shaka Player
+ * Copyright 2016 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+goog.provide('shaka.cea.TsCeaParser');
+
+goog.require('shaka.cea.ICeaParser');
+goog.require('shaka.cea.SeiProcessor');
+goog.require('shaka.util.BufferUtils');
+goog.require('shaka.util.TsParser');
+
+/**
+ * MPEG TS CEA parser.
+ * @implements {shaka.cea.ICeaParser}
+ */
+shaka.cea.TsCeaParser = class {
+  /** */
+  constructor() {
+    /**
+     * SEI data processor.
+     * @private
+     * @const {!shaka.cea.SeiProcessor}
+     */
+    this.seiProcessor_ = new shaka.cea.SeiProcessor();
+  }
+
+  /**
+   * @override
+   */
+  init(initSegment) {
+    // TS hasn't init segment
+  }
+
+  /**
+   * @override
+   */
+  parse(mediaSegment) {
+    const ICeaParser = shaka.cea.ICeaParser;
+
+    /** @type {!Array<!shaka.cea.ICeaParser.CaptionPacket>} **/
+    const captionPackets = [];
+
+    const uint8ArrayData = shaka.util.BufferUtils.toUint8(mediaSegment);
+    if (!shaka.util.TsParser.probe(uint8ArrayData)) {
+      return captionPackets;
+    }
+    const tsParser = new shaka.util.TsParser().parse(uint8ArrayData);
+    const videoNalus = tsParser.getVideoNalus();
+    for (const nalu of videoNalus) {
+      if (nalu.type == ICeaParser.H264_NALU_TYPE_SEI &&
+          nalu.time != null) {
+        for (const packet of this.seiProcessor_.process(nalu.data)) {
+          captionPackets.push({
+            packet: packet,
+            pts: nalu.time,
+          });
+        }
+      }
+    }
+    return captionPackets;
+  }
+};

lib/media/closed_caption_parser.js

@@ -10,6 +10,7 @@ goog.provide('shaka.media.IClosedCaptionParser');
 goog.require('shaka.cea.CeaDecoder');
 goog.require('shaka.cea.DummyCeaParser');
 goog.require('shaka.cea.Mp4CeaParser');
+goog.require('shaka.cea.TsCeaParser');
 goog.require('shaka.util.BufferUtils');
 goog.requireType('shaka.cea.ICaptionDecoder');
 goog.requireType('shaka.cea.ICeaParser');
@@ -59,10 +60,14 @@ shaka.media.ClosedCaptionParser = class {
     /** @private {!shaka.cea.ICeaParser} */
     this.ceaParser_ = new shaka.cea.DummyCeaParser();
 
-    if (mimeType.includes('video/mp4')) {
-      // MP4 Parser to extract closed caption packets from H.264 video.
+    if (mimeType.toLowerCase().includes('video/mp4')) {
+      // MP4 Parser to extract closed caption packets from H.264/H.265 video.
       this.ceaParser_ = new shaka.cea.Mp4CeaParser();
     }
+    if (mimeType.toLowerCase().includes('video/mp2t')) {
+      // TS Parser to extract closed caption packets from H.264 video.
+      this.ceaParser_ = new shaka.cea.TsCeaParser();
+    }
 
     /**
      * Decoder for decoding CEA-X08 data from closed caption packets.