Add config option for using segment relative timestamps for VTT (#542)

* Add config option for using segment relative timestamps for VTT Fix for #480 * Make useRelativeCueTimestamps a non-nullable param * Update tests for the new useRelativeCueTimestamps param * Move period relative timestamp deprecation warning to vtt parser * Log warning only if using absolute timestamps in text cue * Fix vtt text parser test
shaka-project · Oct 25, 2016 · ab43df2 · ab43df2
1 parent 0cb17cb
commit ab43df2
Show file tree

Hide file tree

Showing 19 changed files with 136 additions and 48 deletions.
diff --git a/externs/shaka/player.js b/externs/shaka/player.js
@@ -338,7 +338,8 @@ shakaExtern.ManifestConfiguration;
  *   rebufferingGoal: number,
  *   bufferingGoal: number,
  *   bufferBehind: number,
- *   ignoreTextStreamFailures: boolean
+ *   ignoreTextStreamFailures: boolean,
+ *   useRelativeCueTimestamps: boolean
  * }}
  *
  * @description
@@ -362,6 +363,9 @@ shakaExtern.ManifestConfiguration;
  * @property {boolean} ignoreTextStreamFailures
  *   If true, the player will ignore text stream failures and proceed to play
  *   other streams.
+ * @property {boolean} useRelativeCueTimestamps
+ *   If true, WebVTT cue timestamps will be treated as relative to the start
+ *   time of the VTT segment. Defaults to false.
  * @exportDoc
  */
 shakaExtern.StreamingConfiguration;

diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js
@@ -212,22 +212,28 @@ shaka.media.MediaSourceEngine.prototype.destroy = function() {
  *   MIME types.  For example: { 'audio': 'audio/webm; codecs="vorbis"',
  *   'video': 'video/webm; codecs="vp9"', 'text': 'text/vtt' }.
  *   All types given must be supported.
+ * @param {boolean} useRelativeCueTimestamps
  *
  * @throws InvalidAccessError if blank MIME types are given
  * @throws NotSupportedError if unsupported MIME types are given
  * @throws QuotaExceededError if the browser can't support that many buffers
  *
  * @suppress {unnecessaryCasts}
  */
-shaka.media.MediaSourceEngine.prototype.init = function(typeConfig) {
+shaka.media.MediaSourceEngine.prototype.init =
+    function(typeConfig, useRelativeCueTimestamps) {
+
   for (var contentType in typeConfig) {
     var mimeType = typeConfig[contentType];
     goog.asserts.assert(
         shaka.media.MediaSourceEngine.isTypeSupported(mimeType),
         'Type negotiation should happen before MediaSourceEngine.init!');
 
     if (contentType == 'text') {
-      this.textEngine_ = new shaka.media.TextEngine(this.textTrack_, mimeType);
+      this.textEngine_ =
+          new shaka.media.TextEngine(this.textTrack_,
+                                     mimeType,
+                                     useRelativeCueTimestamps);
     } else {
       var sourceBuffer = this.mediaSource_.addSourceBuffer(mimeType);
       this.eventManager_.listen(

diff --git a/lib/media/mp4_ttml_parser.js b/lib/media/mp4_ttml_parser.js
@@ -31,10 +31,12 @@ goog.require('shaka.util.Mp4Parser');
  * @param {number} offset
  * @param {?number} segmentStartTime
  * @param {?number} segmentEndTime
+ * @param {boolean} useRelativeCueTimestamps Only used by the VTT parser
  * @return {!Array.<!TextTrackCue>}
  */
 shaka.media.Mp4TtmlParser =
-    function(data, offset, segmentStartTime, segmentEndTime) {
+    function(data, offset, segmentStartTime,
+             segmentEndTime, useRelativeCueTimestamps) {
   var reader = new shaka.util.DataViewReader(
       new DataView(data),
       shaka.util.DataViewReader.Endianness.BIG_ENDIAN);
@@ -45,7 +47,7 @@ shaka.media.Mp4TtmlParser =
     // mdat box found, use TtmlTextParser to parse the content
     return shaka.media.TtmlTextParser(
         reader.readBytes(boxSize - 8).buffer, offset,
-        segmentStartTime, segmentEndTime);
+        segmentStartTime, segmentEndTime, false);
   }
   var stppBoxSize = shaka.util.Mp4Parser.findSampleDescriptionBox(
       data, shaka.media.Mp4TtmlParser.BOX_TYPE_STPP);

diff --git a/lib/media/mp4_vtt_parser.js b/lib/media/mp4_vtt_parser.js
@@ -35,10 +35,12 @@ goog.require('shaka.util.TextParser');
  * @param {number} offset
  * @param {?number} segmentStartTime
  * @param {?number} segmentEndTime
+ * @param {boolean} useRelativeCueTimestamps Only used by the VTT parser
  * @return {!Array.<!TextTrackCue>}
  */
 shaka.media.Mp4VttParser =
-    function(data, offset, segmentStartTime, segmentEndTime) {
+    function(data, offset, segmentStartTime,
+             segmentEndTime, useRelativeCueTimestamps) {
   var reader = new shaka.util.DataViewReader(
       new DataView(data),
       shaka.util.DataViewReader.Endianness.BIG_ENDIAN);

diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js
@@ -559,7 +559,8 @@ shaka.media.StreamingEngine.prototype.initStreams_ = function(streamsByType) {
         (stream.codecs ? '; codecs="' + stream.codecs + '"' : '');
   });
 
-  this.mediaSourceEngine_.init(typeConfig);
+  this.mediaSourceEngine_.init(typeConfig,
+                               this.config_.useRelativeCueTimestamps);
   this.setDuration_();
 
   // Setup the initial set of Streams and then begin each update cycle. After

diff --git a/lib/media/text_engine.js b/lib/media/text_engine.js
@@ -30,9 +30,10 @@ goog.require('shaka.util.IDestroyable');
  * @constructor
  * @param {TextTrack} track
  * @param {string} mimeType
+ * @param {boolean} useRelativeCueTimestamps
  * @implements {shaka.util.IDestroyable}
  */
-shaka.media.TextEngine = function(track, mimeType) {
+shaka.media.TextEngine = function(track, mimeType, useRelativeCueTimestamps) {
   /** @private {?shaka.media.TextEngine.TextParser} */
   this.parser_ = shaka.media.TextEngine.parserMap_[mimeType];
 
@@ -54,14 +55,18 @@ shaka.media.TextEngine = function(track, mimeType) {
 
   /** @private {?number} */
   this.bufferEnd_ = null;
+
+  /** @private {boolean} */
+  this.useRelativeCueTimestamps_ = useRelativeCueTimestamps;
 };
 
 
 /**
  * Parses a text buffer into an array of cues.
  *
  * @typedef {
- *   function(ArrayBuffer, number, ?number, ?number):!Array.<!TextTrackCue>
+ *   function(ArrayBuffer, number, ?number,
+ *            ?number, boolean):!Array.<!TextTrackCue>
  * }
  * @exportDoc
  */
@@ -155,7 +160,11 @@ shaka.media.TextEngine.prototype.appendBuffer =
     if (!this.track_) return;
 
     // Parse the buffer and add the new cues.
-    var cues = this.parser_(buffer, offset, startTime, endTime);
+    var cues = this.parser_(buffer,
+                            offset,
+                            startTime,
+                            endTime,
+                            this.useRelativeCueTimestamps_);
 
     if (startTime == null || endTime == null) {
       // Init segments will not have start/end times passed

diff --git a/lib/media/ttml_text_parser.js b/lib/media/ttml_text_parser.js
@@ -30,11 +30,14 @@ goog.require('shaka.util.StringUtils');
  * @param {number} offset
  * @param {?number} segmentStartTime
  * @param {?number} segmentEndTime
+ * @param {boolean} useRelativeCueTimestamps Only used by the VTT parser
  * @return {!Array.<!TextTrackCue>}
  * @throws {shaka.util.Error}
  */
 shaka.media.TtmlTextParser =
-    function(data, offset, segmentStartTime, segmentEndTime) {
+    function(data, offset, segmentStartTime,
+             segmentEndTime, useRelativeCueTimestamps) {
+
   var str = shaka.util.StringUtils.fromUTF8(data);
   var ret = [];
   var parser = new DOMParser();
@@ -49,7 +52,7 @@ shaka.media.TtmlTextParser =
   }
 
   if (xml) {
-    // Try to get the framerate, subRameRate and frameRateMultiplier
+    // Try to get the framerate, subFrameRate and frameRateMultiplier
     // if applicable
     var frameRate = null;
     var subFrameRate = null;

diff --git a/lib/media/vtt_text_parser.js b/lib/media/vtt_text_parser.js
@@ -31,11 +31,20 @@ goog.require('shaka.util.TextParser');
  * @param {number} offset
  * @param {?number} segmentStartTime
  * @param {?number} segmentEndTime
+ * @param {boolean} useRelativeCueTimestamps
  * @return {!Array.<!TextTrackCue>}
  * @throws {shaka.util.Error}
  */
 shaka.media.VttTextParser =
-    function(data, offset, segmentStartTime, segmentEndTime) {
+    function(data, offset, segmentStartTime,
+             segmentEndTime, useRelativeCueTimestamps) {
+
+  if (segmentStartTime > 0 && !useRelativeCueTimestamps) {
+    shaka.log.warning('Period-relative text cue timestamps have been ' +
+                      'deprecated. Segment-relative timestamps will be used ' +
+                      'in V2.1.0 and on.');
+  }
+
   // Get the input as a string.  Normalize newlines to \n.
   var str = shaka.util.StringUtils.fromUTF8(data);
   str = str.replace(/\r\n|\r(?=[^\n]|$)/gm, '\n');
@@ -50,7 +59,10 @@ shaka.media.VttTextParser =
   var ret = [];
   for (var i = 1; i < blocks.length; i++) {
     var lines = blocks[i].split('\n');
-    var cue = shaka.media.VttTextParser.parseCue_(lines, offset);
+    var cue = shaka.media.VttTextParser.parseCue_(lines,
+                                                  offset,
+                                                  segmentStartTime,
+                                                  useRelativeCueTimestamps);
     if (cue)
       ret.push(cue);
   }
@@ -64,10 +76,14 @@ shaka.media.VttTextParser =
  *
  * @param {!Array.<string>} text
  * @param {number} offset
+ * @param {?number} segmentStartTime
+ * @param {boolean} useRelativeCueTimestamps
  * @return {?TextTrackCue}
  * @private
  */
-shaka.media.VttTextParser.parseCue_ = function(text, offset) {
+shaka.media.VttTextParser.parseCue_ =
+    function(text, offset, segmentStartTime, useRelativeCueTimestamps) {
+
   // Skip empty blocks.
   if (text.length == 1 && !text[0])
     return null;
@@ -98,6 +114,12 @@ shaka.media.VttTextParser.parseCue_ = function(text, offset) {
   start += offset;
   end += offset;
 
+  // See issue #480 for discussion on deprecation
+  if (useRelativeCueTimestamps) {
+    start += segmentStartTime;
+    end += segmentStartTime;
+  }
+
   // Get the payload.
   var payload = text.slice(1).join('\n').trim();
 

diff --git a/lib/player.js b/lib/player.js
@@ -1233,7 +1233,8 @@ shaka.Player.prototype.defaultConfig_ = function() {
       rebufferingGoal: 2,
       bufferingGoal: 30,
       bufferBehind: 30,
-      ignoreTextStreamFailures: false
+      ignoreTextStreamFailures: false,
+      useRelativeCueTimestamps: false
     },
     abr: {
       manager: this.defaultAbrManager_,

diff --git a/test/cast/cast_utils_unit.js b/test/cast/cast_utils_unit.js
@@ -167,7 +167,8 @@ describe('CastUtils', function() {
         function onSourceOpen() {
           mediaSourceEngine = new shaka.media.MediaSourceEngine(
               video, mediaSource, /* TextTrack */ null);
-          mediaSourceEngine.init({'video': mimeType});
+
+          mediaSourceEngine.init({'video': mimeType}, false);
           shaka.test.Util.fetch(initSegmentUrl).then(function(data) {
             return mediaSourceEngine.appendBuffer('video', data, null, null);
           }).then(function() {

diff --git a/test/media/cue_integration.js b/test/media/cue_integration.js
@@ -65,6 +65,6 @@ describe('Cue', function() {
    */
   function parseVtt(text, opt_offset) {
     var data = shaka.util.StringUtils.toUTF8(text);
-    return shaka.media.VttTextParser(data, opt_offset || 0, null, null);
+    return shaka.media.VttTextParser(data, opt_offset || 0, null, null, false);
   }
 });
diff --git a/test/media/drm_engine_integration.js b/test/media/drm_engine_integration.js
@@ -119,10 +119,11 @@ describe('DrmEngine', function() {
       eventManager.unlisten(mediaSource, 'sourceopen');
       mediaSourceEngine = new shaka.media.MediaSourceEngine(
           video, mediaSource, null);
+
       mediaSourceEngine.init({
         'video': 'video/mp4; codecs="avc1.640015"',
         'audio': 'audio/mp4; codecs="mp4a.40.2"'
-      });
+      }, false);
       done();
     });
   });

diff --git a/test/media/mp4_ttml_parser_unit.js b/test/media/mp4_ttml_parser_unit.js
@@ -40,21 +40,21 @@ describe('Mp4TtmlParser', function() {
 
   it('parses init segment', function() {
     // Last two parameters are only used by mp4 vtt parser.
-    var ret = shaka.media.Mp4TtmlParser(ttmlInitSegment, 0, null, null);
+    var ret = shaka.media.Mp4TtmlParser(ttmlInitSegment, 0, null, null, false);
     // init segment doesn't have the subtitles. The code should verify
     // their declaration and proceed to the next segment.
     expect(ret).toEqual([]);
   });
 
   it('parses media segment', function() {
-    var ret = shaka.media.Mp4TtmlParser(ttmlSegment, 0, null, null);
+    var ret = shaka.media.Mp4TtmlParser(ttmlSegment, 0, null, null, false);
     expect(ret.length).toBeGreaterThan(0);
   });
 
   it('accounts for offset', function() {
-    var ret1 = shaka.media.Mp4TtmlParser(ttmlSegment, 0, null, null);
+    var ret1 = shaka.media.Mp4TtmlParser(ttmlSegment, 0, null, null, false);
     expect(ret1.length).toBeGreaterThan(0);
-    var ret2 = shaka.media.Mp4TtmlParser(ttmlSegment, 7, null, null);
+    var ret2 = shaka.media.Mp4TtmlParser(ttmlSegment, 7, null, null, false);
     expect(ret2.length).toBeGreaterThan(0);
 
     expect(ret2[0].startTime).toEqual(ret1[0].startTime + 7);
@@ -65,7 +65,7 @@ describe('Mp4TtmlParser', function() {
     var error = new shaka.util.Error(shaka.util.Error.Category.TEXT,
         shaka.util.Error.Code.INVALID_MP4_TTML);
     try {
-      shaka.media.Mp4TtmlParser(audioInitSegment, 0, null, null);
+      shaka.media.Mp4TtmlParser(audioInitSegment, 0, null, null, false);
       fail('Mp4 file with no ttml supported');
     } catch (e) {
       shaka.test.Util.expectToEqualError(e, error);

diff --git a/test/media/mp4_vtt_parser_unit.js b/test/media/mp4_vtt_parser_unit.js
@@ -65,7 +65,7 @@ describe('Mp4vttParser', function() {
   it('parses init segment', function() {
     // init segment doesn't have the subtitles. The code should verify
     // their declaration and proceed to the next segment.
-    var ret = shaka.media.Mp4VttParser(vttInitSegment, 0, null, null);
+    var ret = shaka.media.Mp4VttParser(vttInitSegment, 0, null, null, false);
     expect(ret).toEqual([]);
   });
 
@@ -76,7 +76,7 @@ describe('Mp4vttParser', function() {
          {start: 20, end: 40, text:
            'You\'re a fool for traveling alone,\nso completely unprepared.\n'}
         ];
-    var result = shaka.media.Mp4VttParser(vttSegment, 0, 20, 40);
+    var result = shaka.media.Mp4VttParser(vttSegment, 0, 20, 40, false);
     verifyHelper(cues, result);
   });
 
@@ -89,7 +89,7 @@ describe('Mp4vttParser', function() {
            'You\'re a fool for traveling alone,\nso completely unprepared.\n',
            vertical: 'lr', line: 1}
         ];
-    var result = shaka.media.Mp4VttParser(vttSegSettings, 0, 20, 40);
+    var result = shaka.media.Mp4VttParser(vttSegSettings, 0, 20, 40, false);
     verifyHelper(cues, result);
   });
 
@@ -100,15 +100,15 @@ describe('Mp4vttParser', function() {
          {start: 27, end: 47, text:
            'You\'re a fool for traveling alone,\nso completely unprepared.\n'}
         ];
-    var result = shaka.media.Mp4VttParser(vttSegment, 7, 20, 40);
+    var result = shaka.media.Mp4VttParser(vttSegment, 7, 20, 40, false);
     verifyHelper(cues, result);
   });
 
   it('rejects init segment with no vtt', function() {
     var error = new shaka.util.Error(shaka.util.Error.Category.TEXT,
         shaka.util.Error.Code.INVALID_MP4_VTT);
     try {
-      shaka.media.Mp4VttParser(audioInitSegment, 0, 20, 40);
+      shaka.media.Mp4VttParser(audioInitSegment, 0, 20, 40, false);
       fail('Mp4 file with no vtt supported');
     } catch (e) {
       shaka.test.Util.expectToEqualError(e, error);

diff --git a/test/media/streaming_engine_integration.js b/test/media/streaming_engine_integration.js
@@ -287,7 +287,8 @@ describe('StreamingEngine', function() {
       bufferingGoal: 5,
       retryParameters: shaka.net.NetworkingEngine.defaultRetryParameters(),
       bufferBehind: 15,
-      ignoreTextStreamFailures: false
+      ignoreTextStreamFailures: false,
+      useRelativeCueTimestamps: false
     };
     streamingEngine = new shaka.media.StreamingEngine(
         playhead,