From af73139d05e277e55249e99c7293cb14ac505870 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 8 Nov 2017 09:41:15 -0800 Subject: [PATCH] Set appendWindowStart in MediaSource This avoids having media from one period replaced by media from the next period. Instead, media that comes before the period start will be chopped off by MediaSource. Closes #1098 Change-Id: Idf6dc2ffafe78214e94bc75aca63920e153f1a2c --- lib/media/media_source_engine.js | 34 ++++++++------ lib/media/streaming_engine.js | 26 +++++----- lib/text/text_engine.js | 20 +++++--- test/media/media_source_engine_integration.js | 47 +++++++++++++++++-- test/media/media_source_engine_unit.js | 7 +-- test/text/text_engine_unit.js | 27 ++++++++--- 6 files changed, 114 insertions(+), 47 deletions(-) diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js index 01782e6b85..392ea753eb 100644 --- a/lib/media/media_source_engine.js +++ b/lib/media/media_source_engine.js @@ -522,23 +522,21 @@ shaka.media.MediaSourceEngine.prototype.flush = function(contentType) { * @param {number} timestampOffset The timestamp offset. Segments which start * at time t will be inserted at time t + timestampOffset instead. This * value does not affect segments which have already been inserted. - * @param {?number} appendWindowEnd The timestamp to set the append window end + * @param {number} appendWindowStart The timestamp to set the append window + * start to. Media before this value will be truncated. + * @param {number} appendWindowEnd The timestamp to set the append window end * to. Media beyond this value will be truncated. * @return {!Promise} */ shaka.media.MediaSourceEngine.prototype.setStreamProperties = function( - contentType, timestampOffset, appendWindowEnd) { + contentType, timestampOffset, appendWindowStart, appendWindowEnd) { var ContentType = shaka.util.ManifestParserUtils.ContentType; if (contentType == ContentType.TEXT) { this.textEngine_.setTimestampOffset(timestampOffset); - if (appendWindowEnd != null) - this.textEngine_.setAppendWindowEnd(appendWindowEnd); + this.textEngine_.setAppendWindow(appendWindowStart, appendWindowEnd); return Promise.resolve(); } - if (appendWindowEnd == null) - appendWindowEnd = Infinity; - return Promise.all([ // Queue an abort() to help MSE splice together overlapping segments. // We set appendWindowEnd when we change periods in DASH content, and the @@ -548,9 +546,6 @@ shaka.media.MediaSourceEngine.prototype.setStreamProperties = function( // always enter a PARSING_MEDIA_SEGMENT state and we can't change the // timestamp offset. By calling abort(), we reset the state so we can // set it. - // - // Note that abort() resets both appendWindowStart and appendWindowEnd; - // however, we don't use appendWindowStart. this.enqueueOperation_( contentType, this.abort_.bind(this, contentType)), @@ -559,7 +554,8 @@ shaka.media.MediaSourceEngine.prototype.setStreamProperties = function( this.setTimestampOffset_.bind(this, contentType, timestampOffset)), this.enqueueOperation_( contentType, - this.setAppendWindowEnd_.bind(this, contentType, appendWindowEnd)) + this.setAppendWindow_.bind( + this, contentType, appendWindowStart, appendWindowEnd)) ]); }; @@ -657,14 +653,16 @@ shaka.media.MediaSourceEngine.prototype.remove_ = * @private */ shaka.media.MediaSourceEngine.prototype.abort_ = function(contentType) { - // Save the append window end, which is reset on abort(). + // Save the append window, which is reset on abort(). + var appendWindowStart = this.sourceBuffers_[contentType].appendWindowStart; var appendWindowEnd = this.sourceBuffers_[contentType].appendWindowEnd; // This will not trigger an 'updateend' event, since nothing is happening. // This is only to reset MSE internals, not to abort an actual operation. this.sourceBuffers_[contentType].abort(); - // Restore the append window end. + // Restore the append window. + this.sourceBuffers_[contentType].appendWindowStart = appendWindowStart; this.sourceBuffers_[contentType].appendWindowEnd = appendWindowEnd; // Fake 'updateend' event to resolve the operation. @@ -711,12 +709,18 @@ shaka.media.MediaSourceEngine.prototype.setTimestampOffset_ = /** * Set the SourceBuffer's append window end. * @param {shaka.util.ManifestParserUtils.ContentType} contentType + * @param {number} appendWindowStart * @param {number} appendWindowEnd * @private */ -shaka.media.MediaSourceEngine.prototype.setAppendWindowEnd_ = - function(contentType, appendWindowEnd) { +shaka.media.MediaSourceEngine.prototype.setAppendWindow_ = + function(contentType, appendWindowStart, appendWindowEnd) { + // You can't set start > end, so first set start to 0, then set the new end, + // then set the new start. That way, there are no intermediate states which + // are invalid. + this.sourceBuffers_[contentType].appendWindowStart = 0; this.sourceBuffers_[contentType].appendWindowEnd = appendWindowEnd; + this.sourceBuffers_[contentType].appendWindowStart = appendWindowStart; // Fake 'updateend' event to resolve the operation. this.onUpdateEnd_(contentType); diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js index 6159ae49da..6987d45e41 100644 --- a/lib/media/streaming_engine.js +++ b/lib/media/streaming_engine.js @@ -1337,20 +1337,17 @@ shaka.media.StreamingEngine.prototype.fetchAndAppend_ = function( // callbacks too. var stream = mediaState.stream; - // Compute the append window end. + // Compute the append window. + var duration = this.manifest_.presentationTimeline.getDuration(); var followingPeriod = this.manifest_.periods[currentPeriodIndex + 1]; - var appendWindowEnd = null; - if (followingPeriod) { - appendWindowEnd = followingPeriod.startTime; - } else { - appendWindowEnd = this.manifest_.presentationTimeline.getDuration(); - } + var appendWindowStart = currentPeriod.startTime; + var appendWindowEnd = followingPeriod ? followingPeriod.startTime : duration; goog.asserts.assert( - (appendWindowEnd == null) || (reference.startTime <= appendWindowEnd), + reference.startTime <= appendWindowEnd, logPrefix + ' segment should start before append window end'); - var initSourceBuffer = - this.initSourceBuffer_(mediaState, currentPeriodIndex, appendWindowEnd); + var initSourceBuffer = this.initSourceBuffer_( + mediaState, currentPeriodIndex, appendWindowStart, appendWindowEnd); mediaState.performingUpdate = true; @@ -1523,12 +1520,13 @@ shaka.media.StreamingEngine.prototype.handleQuotaExceeded_ = function( * * @param {shaka.media.StreamingEngine.MediaState_} mediaState * @param {number} currentPeriodIndex - * @param {?number} appendWindowEnd + * @param {number} appendWindowStart + * @param {number} appendWindowEnd * @return {!Promise} * @private */ shaka.media.StreamingEngine.prototype.initSourceBuffer_ = function( - mediaState, currentPeriodIndex, appendWindowEnd) { + mediaState, currentPeriodIndex, appendWindowStart, appendWindowEnd) { if (!mediaState.needInitSegment) return Promise.resolve(); @@ -1543,10 +1541,12 @@ shaka.media.StreamingEngine.prototype.initSourceBuffer_ = function( var timestampOffset = currentPeriod.startTime - mediaState.stream.presentationTimeOffset; shaka.log.v1(logPrefix, 'setting timestamp offset to ' + timestampOffset); + shaka.log.v1(logPrefix, + 'setting appstart window start to ' + appendWindowStart); shaka.log.v1(logPrefix, 'setting append window end to ' + appendWindowEnd); var setStreamProperties = this.playerInterface_.mediaSourceEngine.setStreamProperties( - mediaState.type, timestampOffset, appendWindowEnd); + mediaState.type, timestampOffset, appendWindowStart, appendWindowEnd); if (!mediaState.stream.initSegmentReference) { // The Stream is self initializing. diff --git a/lib/text/text_engine.js b/lib/text/text_engine.js index 8f68e956d9..3121c63951 100644 --- a/lib/text/text_engine.js +++ b/lib/text/text_engine.js @@ -42,6 +42,9 @@ shaka.text.TextEngine = function(displayer) { /** @private {number} */ this.timestampOffset_ = 0; + /** @private {number} */ + this.appendWindowStart_ = 0; + /** @private {number} */ this.appendWindowEnd_ = Infinity; @@ -190,7 +193,8 @@ shaka.text.TextEngine.prototype.appendBuffer = // Parse the buffer and add the new cues. var allCues = this.parser_.parseMedia(new Uint8Array(buffer), time); var cuesToAppend = allCues.filter(function(cue) { - return cue.startTime < this.appendWindowEnd_; + return cue.startTime >= this.appendWindowStart_ && + cue.startTime < this.appendWindowEnd_; }.bind(this)); this.displayer_.append(cuesToAppend); @@ -200,7 +204,7 @@ shaka.text.TextEngine.prototype.appendBuffer = // parsed cues. This is important because some segments may contain no // cues, but we must still consider those ranges buffered. if (this.bufferStart_ == null) { - this.bufferStart_ = startTime; + this.bufferStart_ = Math.max(startTime, this.appendWindowStart_); } else { // We already had something in buffer, and we assume we are extending the // range from the end. @@ -258,10 +262,14 @@ shaka.text.TextEngine.prototype.setTimestampOffset = }; -/** @param {number} windowEnd */ -shaka.text.TextEngine.prototype.setAppendWindowEnd = - function(windowEnd) { - this.appendWindowEnd_ = windowEnd; +/** + * @param {number} appendWindowStart + * @param {number} appendWindowEnd + */ +shaka.text.TextEngine.prototype.setAppendWindow = + function(appendWindowStart, appendWindowEnd) { + this.appendWindowStart_ = appendWindowStart; + this.appendWindowEnd_ = appendWindowEnd; }; diff --git a/test/media/media_source_engine_integration.js b/test/media/media_source_engine_integration.js index e4751cc23f..4bef40c110 100644 --- a/test/media/media_source_engine_integration.js +++ b/test/media/media_source_engine_integration.js @@ -313,7 +313,7 @@ describe('MediaSourceEngine', function() { }).catch(fail).then(done); }); - it('trims content at appendWindowEnd', function(done) { + it('trims content at the append window', function(done) { // Create empty object first and initialize the fields through // [] to allow field names to be expressions. var initObject = {}; @@ -324,15 +324,56 @@ describe('MediaSourceEngine', function() { }).then(function() { return mediaSourceEngine.setStreamProperties(ContentType.VIDEO, /* timestampOffset */ 0, + /* appendWindowStart */ 5, /* appendWindowEnd */ 18); }).then(function() { expect(buffered(ContentType.VIDEO, 0)).toBe(0); return append(ContentType.VIDEO, 1); }).then(function() { - expect(buffered(ContentType.VIDEO, 0)).toBeCloseTo(10); + expect(bufferStart(ContentType.VIDEO)).toBeCloseTo(5, 1); + expect(buffered(ContentType.VIDEO, 5)).toBeCloseTo(5, 1); + return append(ContentType.VIDEO, 2); + }).then(function() { + expect(buffered(ContentType.VIDEO, 5)).toBeCloseTo(13, 1); + }).catch(fail).then(done); + }); + + it('does not remove when overlap is outside append window', function(done) { + // Create empty object first and initialize the fields through + // [] to allow field names to be expressions. + var initObject = {}; + initObject[ContentType.VIDEO] = getFakeStream(metadata.video); + mediaSourceEngine.init(initObject); + mediaSourceEngine.setDuration(presentationDuration).then(function() { + return appendInit(ContentType.VIDEO); + }).then(function() { + // Simulate period 1, with 20 seconds of content, no timestamp offset + return mediaSourceEngine.setStreamProperties(ContentType.VIDEO, + /* timestampOffset */ 0, + /* appendWindowStart */ 0, + /* appendWindowEnd */ 20); + }).then(function() { + return append(ContentType.VIDEO, 1); + }).then(function() { + return append(ContentType.VIDEO, 2); + }).then(function() { + expect(bufferStart(ContentType.VIDEO)).toBeCloseTo(0, 1); + expect(buffered(ContentType.VIDEO, 0)).toBeCloseTo(20, 1); + + // Simulate period 2, with 20 seconds of content offset back by 5 seconds. + // The 5 seconds of overlap should be trimmed off, and we should still + // have a continuous stream with 35 seconds of content. + return mediaSourceEngine.setStreamProperties(ContentType.VIDEO, + /* timestampOffset */ 15, + /* appendWindowStart */ 20, + /* appendWindowEnd */ 35); + }).then(function() { + return append(ContentType.VIDEO, 1); + }).then(function() { return append(ContentType.VIDEO, 2); }).then(function() { - expect(buffered(ContentType.VIDEO, 0)).toBeCloseTo(18, 1); + expect(bufferStart(ContentType.VIDEO)).toBeCloseTo(0, 1); + expect(buffered(ContentType.VIDEO, 0)).toBeCloseTo(35, 1); }).catch(fail).then(done); }); }); diff --git a/test/media/media_source_engine_unit.js b/test/media/media_source_engine_unit.js index eca2db1513..4740678de3 100644 --- a/test/media/media_source_engine_unit.js +++ b/test/media/media_source_engine_unit.js @@ -592,14 +592,15 @@ describe('MediaSourceEngine', function() { it('will forward to TextEngine', function(done) { expect(mockTextEngine.setTimestampOffset).not.toHaveBeenCalled(); - expect(mockTextEngine.setAppendWindowEnd).not.toHaveBeenCalled(); + expect(mockTextEngine.setAppendWindow).not.toHaveBeenCalled(); mediaSourceEngine .setStreamProperties(ContentType.TEXT, /* timestampOffset */ 10, + /* appendWindowStart */ 0, /* appendWindowEnd */ 20) .then(function() { expect(mockTextEngine.setTimestampOffset).toHaveBeenCalledWith(10); - expect(mockTextEngine.setAppendWindowEnd).toHaveBeenCalledWith(20); + expect(mockTextEngine.setAppendWindow).toHaveBeenCalledWith(0, 20); }) .catch(fail) .then(done); @@ -982,7 +983,7 @@ describe('MediaSourceEngine', function() { expect(mockTextEngine).toBeFalsy(); mockTextEngine = jasmine.createSpyObj('TextEngine', [ 'initParser', 'destroy', 'appendBuffer', 'remove', 'setTimestampOffset', - 'setAppendWindowEnd', 'bufferStart', 'bufferEnd', 'bufferedAheadOf', + 'setAppendWindow', 'bufferStart', 'bufferEnd', 'bufferedAheadOf', 'setDisplayer' ]); diff --git a/test/text/text_engine_unit.js b/test/text/text_engine_unit.js index a412d0da80..6b047c826b 100644 --- a/test/text/text_engine_unit.js +++ b/test/text/text_engine_unit.js @@ -275,7 +275,7 @@ describe('TextEngine', function() { }); }); - describe('setAppendWindowEnd', function() { + describe('setAppendWindow', function() { beforeEach(function() { mockParseMedia.and.callFake(function() { return [createFakeCue(0, 1), createFakeCue(1, 2), createFakeCue(2, 3)]; @@ -283,7 +283,7 @@ describe('TextEngine', function() { }); it('limits appended cues', function(done) { - textEngine.setAppendWindowEnd(1.9); + textEngine.setAppendWindow(0, 1.9); textEngine.appendBuffer(dummyData, 0, 3).then(function() { expect(mockDisplayer.append).toHaveBeenCalledWith( [ @@ -292,29 +292,42 @@ describe('TextEngine', function() { ]); mockDisplayer.append.calls.reset(); - textEngine.setAppendWindowEnd(2.1); + textEngine.setAppendWindow(1, 2.1); return textEngine.appendBuffer(dummyData, 0, 3); }).then(function() { expect(mockDisplayer.append).toHaveBeenCalledWith( [ - createFakeCue(0, 1), createFakeCue(1, 2), createFakeCue(2, 3) ]); }).catch(fail).then(done); }); + it('limits bufferStart', function(done) { + textEngine.setAppendWindow(1, 9); + textEngine.appendBuffer(dummyData, 0, 3).then(function() { + expect(textEngine.bufferStart()).toBe(1); + + return textEngine.remove(0, 9); + }).then(function() { + textEngine.setAppendWindow(2.1, 9); + return textEngine.appendBuffer(dummyData, 0, 3); + }).then(function() { + expect(textEngine.bufferStart()).toBe(2.1); + }).catch(fail).then(done); + }); + it('limits bufferEnd', function(done) { - textEngine.setAppendWindowEnd(1.9); + textEngine.setAppendWindow(0, 1.9); textEngine.appendBuffer(dummyData, 0, 3).then(function() { expect(textEngine.bufferEnd()).toBe(1.9); - textEngine.setAppendWindowEnd(2.1); + textEngine.setAppendWindow(0, 2.1); return textEngine.appendBuffer(dummyData, 0, 3); }).then(function() { expect(textEngine.bufferEnd()).toBe(2.1); - textEngine.setAppendWindowEnd(4.1); + textEngine.setAppendWindow(0, 4.1); return textEngine.appendBuffer(dummyData, 0, 3); }).then(function() { expect(textEngine.bufferEnd()).toBe(3);