From a3d6186dd4f063a15f14b93034424adf485af09a Mon Sep 17 00:00:00 2001 From: Sandra Lokshina Date: Fri, 10 Mar 2017 16:39:58 -0800 Subject: [PATCH] Add support for multiple media tags with the same group-id (HLS). Currently HLS parser expects only one media tag to have a given group id. According to the spec that might not be the case. This change adds support for multiple tags with the same gruop id and insures the parser creates variants for all of them. Issue #279. Change-Id: I327e52387f7513464fc56c4b6b8d07ead689d6cc --- lib/hls/hls_parser.js | 262 ++++++++++++++++++++++-------------- lib/hls/hls_utils.js | 11 +- test/hls/hls_parser_unit.js | 60 +++++++++ 3 files changed, 225 insertions(+), 108 deletions(-) diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js index c26043dcd2..59929a7052 100644 --- a/lib/hls/hls_parser.js +++ b/lib/hls/hls_parser.js @@ -31,6 +31,8 @@ goog.require('shaka.media.SegmentIndex'); goog.require('shaka.media.SegmentReference'); goog.require('shaka.net.NetworkingEngine'); goog.require('shaka.util.Error'); +goog.require('shaka.util.Functional'); +goog.require('shaka.util.ManifestParserUtils'); @@ -180,12 +182,13 @@ shaka.hls.HlsParser.prototype.parseManifest_ = function(data, uri) { */ shaka.hls.HlsParser.prototype.createPeriod_ = function(playlist) { var Utils = shaka.hls.Utils; + var Functional = shaka.util.Functional; var tags = playlist.tags; // Create Variants for every 'EXT-X-STREAM-INF' tag. var variantTags = Utils.filterTagsByName(tags, 'EXT-X-STREAM-INF'); var variantsPromises = variantTags.map(function(tag) { - return this.createVariant_(tag, playlist); + return this.createVariantsForTag_(tag, playlist); }.bind(this)); var mediaTags = Utils.filterTagsByName(playlist.tags, 'EXT-X-MEDIA'); @@ -193,14 +196,15 @@ shaka.hls.HlsParser.prototype.createPeriod_ = function(playlist) { var type = this.getRequiredAttributeValue_(tag, 'TYPE'); return type == 'SUBTITLES'; }.bind(this)); - // TODO: CLOSED-CAPTIONS requires the parsing of CEA-608 from the video. + // TODO: CLOSED-CAPTIONS requires the parsing of CEA-608 from the video. var textStreamPromises = textStreamTags.map(function(tag) { return this.createTextStream_(tag, playlist); }.bind(this)); - return Promise.all(variantsPromises).then(function(variants) { + return Promise.all(variantsPromises).then(function(allVariants) { return Promise.all(textStreamPromises).then(function(textStreams) { + var variants = allVariants.reduce(Functional.collapseArrays, []); this.fitSegments_(variants); return { startTime: 0, @@ -213,77 +217,17 @@ shaka.hls.HlsParser.prototype.createPeriod_ = function(playlist) { /** - * Parses an EXT-X-STREAM-INF tag into a Variant. - * * @param {!shaka.hls.Tag} tag * @param {!shaka.hls.Playlist} playlist - * @return {!Promise.} + * @return {!Promise.>} * @private */ -shaka.hls.HlsParser.prototype.createVariant_ = function(tag, playlist) { +shaka.hls.HlsParser.prototype.createVariantsForTag_ = function(tag, playlist) { goog.asserts.assert(tag.name == 'EXT-X-STREAM-INF', 'Should only be called on variant tags!'); - var ContentType = shaka.util.ManifestParserUtils.ContentType; - var bandwidth = Number(this.getRequiredAttributeValue_(tag, 'BANDWIDTH')); - // TODO(ismena): Implement support for protected content. - var drmInfos = []; - - return this.createStreamsForVariant_(tag, playlist) - .then(function(streamsByType) { - var audio = streamsByType[ContentType.AUDIO]; - var video = streamsByType[ContentType.VIDEO]; - return { - id: this.globalId_++, - language: audio ? audio.language : 'und', - primary: (!!audio && audio.primary) || (!!video && video.primary), - audio: audio, - video: video, - bandwidth: bandwidth, - drmInfos: drmInfos, - allowedByApplication: true, - allowedByKeySystem: true - }; - }.bind(this)); -}; - - -/** - * Parses an EXT-X-MEDIA tag with TYPE="SUBTITLES" into a text stream. - * - * @param {!shaka.hls.Tag} tag - * @param {!shaka.hls.Playlist} playlist - * @return {!Promise.} - * @private - */ -shaka.hls.HlsParser.prototype.createTextStream_ = function(tag, playlist) { - goog.asserts.assert(tag.name == 'EXT-X-MEDIA', - 'Should only be called on media tags!'); - - var type = this.getRequiredAttributeValue_(tag, 'TYPE'); - goog.asserts.assert(type == 'SUBTITLES', - 'Should only be called on tags with TYPE="SUBTITLES"!'); - - var timeOffset = this.getTimeOffset_(playlist); - return this.createStreamFromMediaTag_(tag, [], timeOffset); -}; - - -/** - * Creates audio and video streams for the Variant. - * - * @param {!shaka.hls.Tag} tag - * @param {!shaka.hls.Playlist} playlist - * @return {!Promise.>} - * @private - */ -shaka.hls.HlsParser.prototype.createStreamsForVariant_ = - function(tag, playlist) { var Utils = shaka.hls.Utils; - var ContentType = shaka.util.ManifestParserUtils.ContentType; - var streamsByType = {}; - var videoTag = null; - var audioTag = null; + var bandwidth = Number(this.getRequiredAttributeValue_(tag, 'BANDWIDTH')); var codecs = this.getRequiredAttributeValue_(tag, 'CODECS').split(','); var resolutionAttr = tag.getAttribute('RESOLUTION'); @@ -308,54 +252,172 @@ shaka.hls.HlsParser.prototype.createStreamsForVariant_ = var audioAttr = tag.getAttribute('AUDIO'); var videoAttr = tag.getAttribute('VIDEO'); - if (audioAttr) - audioTag = Utils.findMediaTag(mediaTags, 'AUDIO', audioAttr.value); + var audioPromises = []; + var videoPromises = []; - if (videoAttr) - videoTag = Utils.findMediaTag(mediaTags, 'VIDEO', videoAttr.value); + // TODO: support for protected content + var drmInfos = []; - return this.createStreamFromMediaTag_(audioTag, codecs, timeOffset) - .then(function(stream) { - streamsByType[ContentType.AUDIO] = stream; - return this.createStreamFromMediaTag_(videoTag, codecs, timeOffset); - }.bind(this)).then(function(stream) { - this.addVideoAttributes_(stream, width, height, frameRate); - streamsByType[ContentType.VIDEO] = stream; - var audio = streamsByType[ContentType.AUDIO]; - var video = streamsByType[ContentType.VIDEO]; + if (audioAttr) { + var audioTags = Utils.findMediaTags(mediaTags, 'AUDIO', audioAttr.value); + audioPromises = audioTags.map(function(tag) { + return this.createStreamFromMediaTag_(tag, codecs, timeOffset); + }.bind(this)); + } + + if (videoAttr) { + var videoTags = Utils.findMediaTags(mediaTags, 'VIDEO', videoAttr.value); + videoPromises = videoTags.map(function(tag) { + return this.createStreamFromMediaTag_(tag, codecs, timeOffset); + }.bind(this)); + } + + var audioStreams = []; + var videoStreams = []; + + return Promise + .all([ + Promise.all(audioPromises), + Promise.all(videoPromises) + ]) + .then(function(data) { + audioStreams = data[0]; + videoStreams = data[1]; // TODO: find examples of audio-only variants and account // for them. - if (video && audio) { - // Both audio and video streams have already been created. - return streamsByType; - } else if (video && !audio) { - // If video stream has been described by a media tag, + if (videoStreams.length && audioStreams.length) { + return null; + } else if (videoStreams.length && !audioStreams.length) { + // If video streams have been described by a media tag, // assume the underlying uri describes audio. - return this.createStreamFromVariantTag_(tag, codecs, - ContentType.AUDIO, - timeOffset) - .then(function(stream) { - streamsByType[ContentType.AUDIO] = stream; - return streamsByType; - }); + return this.createStreamFromVariantTag_( + tag, codecs, + ContentType.AUDIO, + timeOffset); } else { // In any other case (video-only variants, multiplexed // content audio described by a media tag) assume the // underlying uri describes video. - return this.createStreamFromVariantTag_(tag, codecs, - ContentType.VIDEO, - timeOffset) - .then(function(stream) { - streamsByType[ContentType.VIDEO] = stream; - this.addVideoAttributes_( - stream, width, height, frameRate); - return streamsByType; - }.bind(this)); + return this.createStreamFromVariantTag_( + tag, codecs, + ContentType.VIDEO, + timeOffset); + } + }.bind(this)) + .then(function(stream) { + if (stream) { + if (stream.type == ContentType.AUDIO) + audioStreams = [stream]; + else + videoStreams = [stream]; } + + return this.createVariants_(audioStreams, + videoStreams, + bandwidth, + drmInfos, + width, + height, + frameRate); }.bind(this)); }; +/** + * @param {!Array.} audioStreams + * @param {!Array.} videoStreams + * @param {number} bandwidth + * @param {!Array.} drmInfos + * @param {!string|undefined} width + * @param {!string|undefined} height + * @param {!string|undefined} frameRate + * @return {!Array.} + * @private + */ +shaka.hls.HlsParser.prototype.createVariants_ = + function(audioStreams, videoStreams, bandwidth, drmInfos, + width, height, frameRate) { + + videoStreams.forEach(function(stream) { + this.addVideoAttributes_(stream, width, height, frameRate); + }.bind(this)); + + // In case of audio-only or video-only content, we create an array of + // one item containing a null. This way, the double-loop works for all + // kinds of content. + // NOTE: we currently don't have support for audio-only content. + if (!videoStreams.length) + videoStreams = [null]; + if (!audioStreams.length) + audioStreams = [null]; + + var variants = []; + for (var i = 0; i < audioStreams.length; i++) { + for (var j = 0; j < videoStreams.length; j++) { + var audio = audioStreams[i]; + var video = videoStreams[j]; + variants.push(this.createVariant_( + audio, video, bandwidth, drmInfos)); + } + } + return variants; +}; + + +/** + * @param {shakaExtern.Stream} audio + * @param {shakaExtern.Stream} video + * @param {number} bandwidth + * @param {!Array.} drmInfos + * @return {!shakaExtern.Variant} + * @private + */ +shaka.hls.HlsParser.prototype.createVariant_ = + function(audio, video, bandwidth, drmInfos) { + var ContentType = shaka.util.ManifestParserUtils.ContentType; + + // Since both audio and video are of the same type, this assertion will catch + // certain mistakes at runtime that the compiler would miss. + goog.asserts.assert(!audio || audio.type == ContentType.AUDIO, + 'Audio parameter mismatch!'); + goog.asserts.assert(!video || video.type == ContentType.VIDEO, + 'Video parameter mismatch!'); + + return { + id: this.globalId_++, + language: audio ? audio.language : 'und', + primary: (!!audio && audio.primary) || (!!video && video.primary), + audio: audio, + video: video, + bandwidth: bandwidth, + drmInfos: drmInfos, + allowedByApplication: true, + allowedByKeySystem: true + }; +}; + + +/** + * Parses an EXT-X-MEDIA tag with TYPE="SUBTITLES" into a text stream. + * + * @param {!shaka.hls.Tag} tag + * @param {!shaka.hls.Playlist} playlist + * @return {!Promise.} + * @private + */ +shaka.hls.HlsParser.prototype.createTextStream_ = function(tag, playlist) { + goog.asserts.assert(tag.name == 'EXT-X-MEDIA', + 'Should only be called on media tags!'); + + var type = this.getRequiredAttributeValue_(tag, 'TYPE'); + goog.asserts.assert(type == 'SUBTITLES', + 'Should only be called on tags with TYPE="SUBTITLES"!'); + + var timeOffset = this.getTimeOffset_(playlist); + return this.createStreamFromMediaTag_(tag, [], timeOffset); +}; + + /** * Parse EXT-X-MEDIA media tag into a Stream object. * diff --git a/lib/hls/hls_utils.js b/lib/hls/hls_utils.js index efbe9d1f94..4724f818a4 100644 --- a/lib/hls/hls_utils.js +++ b/lib/hls/hls_utils.js @@ -54,19 +54,14 @@ shaka.hls.Utils.getFirstTagWithName = function(tags, name) { * @param {!Array.} tags * @param {!string} type * @param {!string} groupId - * @return {shaka.hls.Tag} + * @return {!Array} */ -shaka.hls.Utils.findMediaTag = function(tags, type, groupId) { - var filtered = tags.filter(function(tag) { +shaka.hls.Utils.findMediaTags = function(tags, type, groupId) { + return tags.filter(function(tag) { var typeAttr = tag.getAttribute('TYPE'); var groupIdAttr = tag.getAttribute('GROUP-ID'); return typeAttr.value == type && groupIdAttr.value == groupId; }); - - if (filtered.length) - return filtered[0]; - else - return null; }; diff --git a/test/hls/hls_parser_unit.js b/test/hls/hls_parser_unit.js index 7918807045..6cba6187a0 100644 --- a/test/hls/hls_parser_unit.js +++ b/test/hls/hls_parser_unit.js @@ -201,6 +201,66 @@ describe('HlsParser', function() { testHlsParser(master, media, manifest, done); }); + it('parses multiple streams with the same group id', function(done) { + var master = [ + '#EXTM3U\n', + '#EXT-X-STREAM-INF:BANDWIDTH=200,CODECS="avc1,mp4a",', + 'RESOLUTION=960x540,FRAME-RATE=60,AUDIO="aud1"\n', + 'test://video\n', + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud1",LANGUAGE="en",', + 'URI="test://audio"\n', + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud1",LANGUAGE="fr",', + 'URI="test://audio"\n' + ].join(''); + + var media = [ + '#EXTM3U\n', + '#EXT-X-MAP:URI="test://main.mp4",BYTERANGE="616@0"\n', + '#EXTINF:5,\n', + '#EXT-X-BYTERANGE:121090@616\n', + 'test://main.mp4' + ].join(''); + + var manifest = new shaka.test.ManifestGenerator() + .anyTimeline() + .addPeriod(jasmine.any(Number)) + .addVariant(jasmine.any(Number)) + .language('en') + .bandwidth(200) + .addVideo(jasmine.any(Number)) + .anySegmentFunctions() + .anyInitSegment() + .presentationTimeOffset(0) + .mime('video/mp4', 'avc1') + .frameRate(60) + .size(960, 540) + .addAudio(jasmine.any(Number)) + .language('en') + .anySegmentFunctions() + .anyInitSegment() + .presentationTimeOffset(0) + .mime('audio/mp4', 'mp4a') + .addVariant(jasmine.any(Number)) + .language('fr') + .bandwidth(200) + .addVideo(jasmine.any(Number)) + .anySegmentFunctions() + .anyInitSegment() + .presentationTimeOffset(0) + .mime('video/mp4', 'avc1') + .frameRate(60) + .size(960, 540) + .addAudio(jasmine.any(Number)) + .language('fr') + .anySegmentFunctions() + .anyInitSegment() + .presentationTimeOffset(0) + .mime('audio/mp4', 'mp4a') + .build(); + + testHlsParser(master, media, manifest, done); + }); + it('parses manifest with text streams', function(done) { var master = [ '#EXTM3U\n',