feat(cea): Add CEA parser for TS

README.md

@@ -225,11 +225,9 @@ Shaka Player supports:
   - TTML
     - Supported in both XML form and embedded in MP4
   - CEA-608
-    - Supported embedded in MP4
-    - With help from [mux.js][] v6.2.0+, supported embedded in TS
+    - Supported embedded in MP4 and TS
   - CEA-708
-    - Supported embedded in MP4
-    - With help from [mux.js][] v6.2.0+, supported embedded in TS
+    - Supported embedded in MP4 and TS
   - SubRip (SRT)
     - UTF-8 encoding only
   - LyRiCs (LRC)

build/types/cea

@@ -12,3 +12,4 @@
 +../../lib/cea/i_cea_parser.js
 +../../lib/cea/mp4_cea_parser.js
 +../../lib/cea/sei_processor.js
++../../lib/cea/ts_cea_parser.js

externs/mux.js

@@ -23,29 +23,6 @@ var muxjs = {};
 muxjs.mp4 = {};
 
 
-/** @const */
-muxjs.mp4.probe = class {
-  /**
-   * Parses an MP4 initialization segment and extracts the timescale
-   * values for any declared tracks.
-   *
-   * @param {Uint8Array} init The bytes of the init segment
-   * @return {!Object.<number, number>} a hash of track ids to timescale
-   * values or null if the init segment is malformed.
-   */
-  static timescale(init) {}
-
-  /**
-    * Find the trackIds of the video tracks in this source.
-    * Found by parsing the Handler Reference and Track Header Boxes:
-    *
-    * @param {Uint8Array} init The bytes of the init segment for this source
-    * @return {!Array.<number>} A list of trackIds
-   **/
-  static videoTrackIds(init) {}
-};
-
-
 muxjs.mp4.Transmuxer = class {
   /** @param {Object=} options */
   constructor(options) {}
@@ -74,100 +51,19 @@ muxjs.mp4.Transmuxer = class {
 
   /** Remove all handlers and clean up. */
   dispose() {}
-
-  /** Reset captions. */
-  resetCaptions() {}
 };
 
 
 /**
  * @typedef {{
  *   initSegment: !Uint8Array,
- *   data: !Uint8Array,
- *   captions: !Array
+ *   data: !Uint8Array
  * }}
  *
  * @description Transmuxed data from mux.js.
  * @property {!Uint8Array} initSegment
  * @property {!Uint8Array} data
- * @property {!Array} captions
  * @exportDoc
  */
 muxjs.mp4.Transmuxer.Segment;
 
-
-muxjs.mp4.CaptionParser = class {
-  /**
-   * Parser for CEA closed captions embedded in video streams for Dash.
-   * @constructor
-   * @struct
-   */
-  constructor() {}
-
-  /** Initializes the closed caption parser. */
-  init() {}
-
-  /**
-   * Return true if a new video track is selected or if the timescale is
-   * changed.
-   * @param {!Array.<number>} videoTrackIds A list of video tracks found in the
-   *    init segment.
-   * @param {!Object.<number, number>} timescales The map of track Ids and the
-   *    tracks' timescales in the init segment.
-   * @return {boolean}
-   */
-  isNewInit(videoTrackIds, timescales) {}
-
-  /**
-   * Parses embedded CEA closed captions and interacts with the underlying
-   * CaptionStream, and return the parsed captions.
-   * @param {!Uint8Array} segment The fmp4 segment containing embedded captions
-   * @param {!Array.<number>} videoTrackIds A list of video tracks found in the
-   *    init segment.
-   * @param {!Object.<number, number>} timescales The timescales found in the
-   *    init segment.
-   * @return {muxjs.mp4.ParsedClosedCaptions}
-   */
-  parse(segment, videoTrackIds, timescales) {}
-
-  /** Clear the parsed closed captions data for new data. */
-  clearParsedCaptions() {}
-
-  /** Reset the captions stream. */
-  resetCaptionStream() {}
-};
-
-
-/**
- * @typedef {{
- *   captionStreams: Object.<string, boolean>,
- *   captions: !Array.<muxjs.mp4.ClosedCaption>
- * }}
- *
- * @description closed captions data parsed from mux.js caption parser.
- * @property {Object.<string, boolean>} captionStreams
- * @property {Array.<muxjs.mp4.ClosedCaption>} captions
- */
-muxjs.mp4.ParsedClosedCaptions;
-
-
-/**
- * @typedef {{
- *   startPts: number,
- *   endPts: number,
- *   startTime: number,
- *   endTime: number,
- *   stream: string,
- *   text: string
- * }}
- *
- * @description closed caption parsed from mux.js caption parser.
- * @property {number} startPts
- * @property {number} endPts
- * @property {number} startTime
- * @property {number} endTime
- * @property {string} stream The channel id of the closed caption.
- * @property {string} text The content of the closed caption.
- */
-muxjs.mp4.ClosedCaption;
-

externs/shaka/player.js

@@ -1012,10 +1012,8 @@ shaka.extern.ManifestConfiguration;
  *   the default value unless you have a good reason not to.
  * @property {boolean} forceTransmux
  *   If this is <code>true</code>, we will transmux AAC and TS content even if
- *   not strictly necessary for the assets to be played.  Shaka Player
- *   currently only supports CEA 708 captions by transmuxing, so this value is
- *   necessary for enabling them on platforms with native TS support like Edge
- *   or Chromecast. This value defaults to <code>false</code>.
+ *   not strictly necessary for the assets to be played.
+ *   This value defaults to <code>false</code>.
  * @property {number} safeSeekOffset
  *   The amount of seconds that should be added when repositioning the playhead
  *   after falling out of the availability window or seek. This gives the player

lib/cea/ts_cea_parser.js

@@ -0,0 +1,65 @@
+/*! @license
+ * Shaka Player
+ * Copyright 2016 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+goog.provide('shaka.cea.TsCeaParser');
+
+goog.require('shaka.cea.ICeaParser');
+goog.require('shaka.cea.SeiProcessor');
+goog.require('shaka.util.BufferUtils');
+goog.require('shaka.util.TsParser');
+
+/**
+ * MPEG TS CEA parser.
+ * @implements {shaka.cea.ICeaParser}
+ */
+shaka.cea.TsCeaParser = class {
+  /** */
+  constructor() {
+    /**
+     * SEI data processor.
+     * @private
+     * @const {!shaka.cea.SeiProcessor}
+     */
+    this.seiProcessor_ = new shaka.cea.SeiProcessor();
+  }
+
+  /**
+   * @override
+   */
+  init(initSegment) {
+    // TS hasn't init segment
+  }
+
+  /**
+   * @override
+   */
+  parse(mediaSegment) {
+    const ICeaParser = shaka.cea.ICeaParser;
+
+    /** @type {!Array<!shaka.cea.ICeaParser.CaptionPacket>} **/
+    const captionPackets = [];
+
+    const uint8ArrayData = shaka.util.BufferUtils.toUint8(mediaSegment);
+    if (!shaka.util.TsParser.probe(uint8ArrayData)) {
+      return captionPackets;
+    }
+    const tsParser = new shaka.util.TsParser().parse(uint8ArrayData);
+    const videoNalus = tsParser.getVideoNalus();
+    for (const nalu of videoNalus) {
+      if (nalu.type == ICeaParser.H264_NALU_TYPE_SEI &&
+          nalu.time != null) {
+        for (const packet of this.seiProcessor_
+            .process(nalu.data)) {
+          captionPackets.push({
+            packet: packet,
+            pts: nalu.time,
+          });
+        }
+      }
+    }
+    return captionPackets;
+  }
+};

lib/media/closed_caption_parser.js

@@ -10,6 +10,7 @@ goog.provide('shaka.media.IClosedCaptionParser');
 goog.require('shaka.cea.CeaDecoder');
 goog.require('shaka.cea.DummyCeaParser');
 goog.require('shaka.cea.Mp4CeaParser');
+goog.require('shaka.cea.TsCeaParser');
 goog.require('shaka.util.BufferUtils');
 goog.requireType('shaka.cea.ICaptionDecoder');
 goog.requireType('shaka.cea.ICeaParser');
@@ -59,10 +60,14 @@ shaka.media.ClosedCaptionParser = class {
     /** @private {!shaka.cea.ICeaParser} */
     this.ceaParser_ = new shaka.cea.DummyCeaParser();
 
-    if (mimeType.includes('video/mp4')) {
-      // MP4 Parser to extract closed caption packets from H.264 video.
+    if (mimeType.toLowerCase().includes('video/mp4')) {
+      // MP4 Parser to extract closed caption packets from H.264/H.265 video.
       this.ceaParser_ = new shaka.cea.Mp4CeaParser();
     }
+    if (mimeType.toLowerCase().includes('video/mp2t')) {
+      // TS Parser to extract closed caption packets from H.264 video.
+      this.ceaParser_ = new shaka.cea.TsCeaParser();
+    }
 
     /**
      * Decoder for decoding CEA-X08 data from closed caption packets.

lib/media/media_source_engine.js

@@ -580,6 +580,8 @@ shaka.media.MediaSourceEngine = class {
       return;
     }
 
+    let timestampOffset = this.sourceBuffers_[contentType].timestampOffset;
+
     const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);
     let mimeType = this.sourceBufferTypes_[contentType];
     if (this.transmuxers_[contentType]) {
@@ -590,7 +592,7 @@ shaka.media.MediaSourceEngine = class {
       // The SourceBuffer timestampOffset may or may not be set yet, so this is
       // the timestamp offset that would eventually compute for this segment
       // either way.
-      const timestampOffset =
+      timestampOffset =
           reference.startTime - (tsParser.getStartTime()[contentType] || 0);
       const metadata = tsParser.getMetadata();
       if (metadata.length) {
@@ -613,37 +615,7 @@ shaka.media.MediaSourceEngine = class {
         }
       }
     }
-
-    if (this.transmuxers_[contentType]) {
-      // When seeked we should reset the transmuxer captionstreams
-      // so it does not ignores the captions from previous segments
-      if (seeked) {
-        this.transmuxers_[contentType].resetCaptions();
-      }
-
-      const transmuxedData =
-          await this.transmuxers_[contentType].transmux(data);
-      // For HLS CEA-608/708 CLOSED-CAPTIONS, text data is embedded in
-      // the video stream, so textEngine may not have been initialized.
-      if (!this.textEngine_) {
-        this.reinitText('text/vtt', this.sequenceMode_);
-      }
-      // This doesn't work for native TS support (ex. Edge/Chromecast),
-      // since no transmuxing is needed for native TS.
-      if (transmuxedData.captions && transmuxedData.captions.length) {
-        const videoOffset =
-        this.sourceBuffers_[ContentType.VIDEO].timestampOffset;
-        const closedCaptions = this.textEngine_
-            .convertMuxjsCaptionsToShakaCaptions(transmuxedData.captions);
-        this.textEngine_.storeAndAppendClosedCaptions(
-            closedCaptions,
-            reference ? reference.startTime : null,
-            reference ? reference.endTime : null,
-            videoOffset);
-      }
-
-      data = transmuxedData.data;
-    } else if (hasClosedCaptions && contentType == ContentType.VIDEO) {
+    if (hasClosedCaptions && contentType == ContentType.VIDEO) {
       if (!this.textEngine_) {
         this.reinitText('text/vtt', this.sequenceMode_);
       }
@@ -657,17 +629,19 @@ shaka.media.MediaSourceEngine = class {
       } else {
         const closedCaptions = this.captionParser_.parseFrom(data);
         if (closedCaptions.length) {
-          const videoOffset =
-            this.sourceBuffers_[ContentType.VIDEO].timestampOffset;
           this.textEngine_.storeAndAppendClosedCaptions(
               closedCaptions,
               reference.startTime,
               reference.endTime,
-              videoOffset);
+              timestampOffset);
         }
       }
     }
 
+    if (this.transmuxers_[contentType]) {
+      data = await this.transmuxers_[contentType].transmux(data);
+    }
+
     data = this.workAroundBrokenPlatforms_(
         data, reference ? reference.startTime : null, contentType);
 

lib/media/transmuxer.js

@@ -44,9 +44,6 @@ shaka.media.Transmuxer = class {
     /** @private {!Array.<!Uint8Array>} */
     this.transmuxedData_ = [];
 
-    /** @private {!Array.<muxjs.mp4.ClosedCaption>} */
-    this.captions_ = [];
-
     /** @private {boolean} */
     this.isTransmuxing_ = false;
 
@@ -210,16 +207,14 @@ shaka.media.Transmuxer = class {
   /**
    * Transmux from Transport stream to MP4, using the mux.js library.
    * @param {BufferSource} data
-   * @return {!Promise.<{data: !Uint8Array,
-   *                     captions: !Array.<!muxjs.mp4.ClosedCaption>}>}
+   * @return {!Promise.<!Uint8Array>}
    */
   transmux(data) {
     goog.asserts.assert(!this.isTransmuxing_,
         'No transmuxing should be in progress.');
     this.isTransmuxing_ = true;
     this.transmuxPromise_ = new shaka.util.PublicPromise();
     this.transmuxedData_ = [];
-    this.captions_ = [];
 
     const dataArray = shaka.util.BufferUtils.toUint8(data);
     this.muxTransmuxer_.push(dataArray);
@@ -239,13 +234,6 @@ shaka.media.Transmuxer = class {
     return this.transmuxPromise_;
   }
 
-  /**
-   * Reset captions from Transport stream to MP4, using the mux.js library.
-  */
-  resetCaptions() {
-    this.muxTransmuxer_.resetCaptions();
-  }
-
   /**
    * Handles the 'data' event of the transmuxer.
    * Extracts the cues from the transmuxed segment, and adds them to an array.
@@ -256,7 +244,6 @@ shaka.media.Transmuxer = class {
    * @private
    */
   onTransmuxed_(segment) {
-    this.captions_ = segment.captions;
     this.transmuxedData_.push(
         shaka.util.Uint8ArrayUtils.concat(segment.initSegment, segment.data));
   }
@@ -268,12 +255,8 @@ shaka.media.Transmuxer = class {
    * @private
    */
   onTransmuxDone_() {
-    const output = {
-      data: shaka.util.Uint8ArrayUtils.concat(...this.transmuxedData_),
-      captions: this.captions_,
-    };
-
-    this.transmuxPromise_.resolve(output);
+    const data = shaka.util.Uint8ArrayUtils.concat(...this.transmuxedData_);
+    this.transmuxPromise_.resolve(data);
     this.isTransmuxing_ = false;
   }
 };