From b43b39f79403e89882ec762cd994c4fa9401fae1 Mon Sep 17 00:00:00 2001 From: Alvaro Velad Galvan Date: Fri, 16 Jun 2023 09:19:14 +0200 Subject: [PATCH 1/2] feat(HLS): Optimization of LL-HLS with byterange --- lib/hls/hls_parser.js | 52 +++++++++++++-- lib/hls/manifest_text_parser.js | 10 +-- lib/media/segment_prefetch.js | 8 ++- lib/media/segment_reference.js | 22 +++++++ lib/media/streaming_engine.js | 5 +- test/hls/hls_live_unit.js | 113 ++++++++++++++++++++++++++------ 6 files changed, 174 insertions(+), 36 deletions(-) diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js index c8e26ce720..4f9da44592 100644 --- a/lib/hls/hls_parser.js +++ b/lib/hls/hls_parser.js @@ -2781,6 +2781,7 @@ shaka.hls.HlsParser = class { * @param {!Map.} variables * @param {string} absoluteMediaPlaylistUri * @param {string} type + * @param {string} mimeType * @param {number} timestampOffset * @param {shaka.extern.HlsAes128Key=} hlsAes128Key * @return {shaka.media.SegmentReference} @@ -2788,7 +2789,7 @@ shaka.hls.HlsParser = class { */ createSegmentReference_( initSegmentReference, previousReference, hlsSegment, startTime, - variables, absoluteMediaPlaylistUri, type, timestampOffset, + variables, absoluteMediaPlaylistUri, type, mimeType, timestampOffset, hlsAes128Key) { const tags = hlsSegment.tags; const extinfTag = @@ -2836,8 +2837,18 @@ shaka.hls.HlsParser = class { } // Create SegmentReferences for the partial segments. - const partialSegmentRefs = []; - if (this.lowLatencyMode_) { + let partialSegmentRefs = []; + + // Optimization for LL-HLS with byterange + // More info in https://tinyurl.com/hls-open-byte-range + let segmentWithByteRangeOptimization = false; + let getUrisOptimization = null; + let somePartialSegmentWithGap = false; + + if (this.lowLatencyMode_ && hlsSegment.partialSegments.length) { + const byterangeOptimizationSupport = (mimeType == 'video/mp4' || + mimeType == 'audio/mp4') && window.ReadableStream; + let partialSyncTime = syncTime; for (let i = 0; i < hlsSegment.partialSegments.length; i++) { const item = hlsSegment.partialSegments[i]; @@ -2870,6 +2881,11 @@ shaka.hls.HlsParser = class { const pByterangeLength = item.getAttributeValue('BYTERANGE-LENGTH'); if (pByterangeLength) { pEndByte = pStartByte + Number(pByterangeLength) - 1; + } else if (pStartByte) { + // If we have a non-zero start byte, but no end byte, follow the + // recommendation of https://tinyurl.com/hls-open-byte-range and + // set the end byte explicitly to a large integer. + pEndByte = Number.MAX_SAFE_INTEGER; } } else { const pByterange = item.getAttributeValue('BYTERANGE'); @@ -2884,6 +2900,7 @@ shaka.hls.HlsParser = class { let partialStatus = shaka.media.SegmentReference.Status.AVAILABLE; if (item.getAttributeValue('GAP') == 'YES') { partialStatus = shaka.media.SegmentReference.Status.MISSING; + somePartialSegmentWithGap = true; } let pAbsoluteUri = null; @@ -2896,6 +2913,12 @@ shaka.hls.HlsParser = class { return [pAbsoluteUri]; }; + if (byterangeOptimizationSupport && + pStartByte >= 0 && pEndByte != null) { + getUrisOptimization = getPartialUris; + segmentWithByteRangeOptimization = true; + } + const partial = new shaka.media.SegmentReference( pStartTime, pEndTime, @@ -2943,6 +2966,17 @@ shaka.hls.HlsParser = class { endTime = partialSegmentRefs[partialSegmentRefs.length - 1].endTime; } + if (segmentWithByteRangeOptimization) { + // We can no optimize segments with GAPs or if the start byterange is + // different to 0 + if (somePartialSegmentWithGap || partialSegmentRefs[0].startByte != 0) { + segmentWithByteRangeOptimization = false; + getUrisOptimization = null; + } else { + partialSegmentRefs = []; + } + } + // If the segment has EXT-X-BYTERANGE tag, set the start byte and end byte // base on the byterange information. If segment has no EXT-X-BYTERANGE tag // and has partial segments, set the start byte and end byte base on the @@ -2975,6 +3009,9 @@ shaka.hls.HlsParser = class { let absoluteSegmentUri = null; const getUris = () => { + if (getUrisOptimization) { + return getUrisOptimization(); + } if (absoluteSegmentUri == null) { absoluteSegmentUri = this.variableSubstitution_( hlsSegment.absoluteUri, variables); @@ -2982,7 +3019,7 @@ shaka.hls.HlsParser = class { return absoluteSegmentUri.length ? [absoluteSegmentUri] : []; }; - return new shaka.media.SegmentReference( + const reference = new shaka.media.SegmentReference( startTime, endTime, getUris, @@ -2999,6 +3036,12 @@ shaka.hls.HlsParser = class { status, hlsAes128Key, ); + + if (segmentWithByteRangeOptimization) { + reference.markAsByterangeOptimization(); + } + + return reference; } @@ -3125,6 +3168,7 @@ shaka.hls.HlsParser = class { variables, playlist.absoluteUri, type, + mimeType, lastDiscontinuityStartTime, hlsAes128Key); previousReference = reference; diff --git a/lib/hls/manifest_text_parser.js b/lib/hls/manifest_text_parser.js index 83b9427617..c413c414cc 100644 --- a/lib/hls/manifest_text_parser.js +++ b/lib/hls/manifest_text_parser.js @@ -158,15 +158,7 @@ shaka.hls.ManifestTextParser = class { partialSegmentTags.push(tag); } else if (tag.name == 'EXT-X-PRELOAD-HINT') { if (tag.getAttributeValue('TYPE') == 'PART') { - // Note: BYTERANGE-START without BYTERANGE-LENGTH is being - // ignored. - if (tag.getAttributeValue('BYTERANGE-START') != null) { - if (tag.getAttributeValue('BYTERANGE-LENGTH') != null) { - partialSegmentTags.push(tag); - } - } else { - partialSegmentTags.push(tag); - } + partialSegmentTags.push(tag); } else if (tag.getAttributeValue('TYPE') == 'MAP') { // Rename the Preload Hint tag to be a Map tag. tag.setName('EXT-X-MAP'); diff --git a/lib/media/segment_prefetch.js b/lib/media/segment_prefetch.js index dcf47d7166..573bb5b3f6 100644 --- a/lib/media/segment_prefetch.js +++ b/lib/media/segment_prefetch.js @@ -67,7 +67,13 @@ shaka.media.SegmentPrefetch = class { let reference = startReference; while (this.segmentPrefetchMap_.size < this.prefetchLimit_ && reference != null) { - if (!this.segmentPrefetchMap_.has(reference)) { + // By default doesn't prefech preload partial segments when using + // byterange + let prefetchAllowed = true; + if (reference.isPreload() && reference.endByte != null) { + prefetchAllowed = false; + } + if (prefetchAllowed && !this.segmentPrefetchMap_.has(reference)) { const segmentPrefetchOperation = new shaka.media.SegmentPrefetchOperation(this.fetchDispatcher_); segmentPrefetchOperation.dispatchFetch(reference, this.stream_); diff --git a/lib/media/segment_reference.js b/lib/media/segment_reference.js index 9b020f3590..d7ba5b6486 100644 --- a/lib/media/segment_reference.js +++ b/lib/media/segment_reference.js @@ -268,6 +268,9 @@ shaka.media.SegmentReference = class { /** @type {boolean} */ this.independent = true; + /** @type {boolean} */ + this.byterangeOptimization = false; + /** @type {?shaka.extern.HlsAes128Key} */ this.hlsAes128Key = hlsAes128Key; @@ -429,6 +432,25 @@ shaka.media.SegmentReference = class { return this.independent; } + /** + * Mark the reference as byterange optimization. + * + * @export + */ + markAsByterangeOptimization() { + this.byterangeOptimization = true; + } + + /** + * Returns true if the segment has a byterange optimization. + * + * @return {boolean} + * @export + */ + hasByterangeOptimization() { + return this.byterangeOptimization; + } + /** * Set the segment's thumbnail sprite. * diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js index d654166c90..59a08b735e 100644 --- a/lib/media/streaming_engine.js +++ b/lib/media/streaming_engine.js @@ -1330,9 +1330,10 @@ shaka.media.StreamingEngine = class { stream.mimeType == 'audio/mp4'; const isReadableStreamSupported = window.ReadableStream; // Enable MP4 low latency streaming with ReadableStream chunked data. - // And only for DASH. + // And only for DASH and HLS with byterange optimization. if (this.config_.lowLatencyMode && isReadableStreamSupported && isMP4 && - this.manifest_.type != shaka.media.ManifestParser.HLS) { + (this.manifest_.type != shaka.media.ManifestParser.HLS || + reference.hasByterangeOptimization())) { let remaining = new Uint8Array(0); let processingResult = false; let callbackCalled = false; diff --git a/test/hls/hls_live_unit.js b/test/hls/hls_live_unit.js index 5c0d61a259..7e867ec33b 100644 --- a/test/hls/hls_live_unit.js +++ b/test/hls/hls_live_unit.js @@ -634,58 +634,132 @@ describe('HlsParser live', () => { '#EXTM3U\n', '#EXT-X-TARGETDURATION:5\n', '#EXT-X-PART-INF:PART-TARGET=1.5\n', - '#EXT-X-MAP:URI="init.mp4",BYTERANGE="616@0"\n', + '#EXT-X-MAP:URI="init.mp4"\n', '#EXT-X-MEDIA-SEQUENCE:0\n', // ref includes partialRef, partialRef2 // partialRef - '#EXT-X-PART:DURATION=2,URI="partial.mp4",BYTERANGE=200@0,', - 'INDEPENDENT=YES\n', + '#EXT-X-PART:DURATION=2,URI="partial.mp4",INDEPENDENT=YES\n', // partialRef2 - '#EXT-X-PART:DURATION=2,URI="partial2.mp4",BYTERANGE=230@200,', - 'INDEPENDENT=YES\n', + '#EXT-X-PART:DURATION=2,URI="partial2.mp4",INDEPENDENT=YES\n', '#EXTINF:4,\n', 'main.mp4\n', // ref2 includes partialRef3, preloadRef // partialRef3 - '#EXT-X-PART:DURATION=2,URI="partial.mp4",BYTERANGE=210@0,', - 'INDEPENDENT=YES\n', + '#EXT-X-PART:DURATION=2,URI="partial.mp4",INDEPENDENT=YES\n', // preloadRef - '#EXT-X-PRELOAD-HINT:TYPE=PART,URI="partial.mp4",BYTERANGE-START=210,', - 'BYTERANGE-LENGTH=210\n', + '#EXT-X-PRELOAD-HINT:TYPE=PART,URI="partial.mp4"\n', ].join(''); const partialRef = makeReference( 'test:/partial.mp4', 0, 2, /* syncTime= */ null, - /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 199); + /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null); const partialRef2 = makeReference( 'test:/partial2.mp4', 2, 4, /* syncTime= */ null, - /* baseUri= */ '', /* startByte= */ 200, /* endByte= */ 429); + /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null); const ref = makeReference( 'test:/main.mp4', 0, 4, /* syncTime= */ null, - /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 429, + /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null, /* timestampOffset= */ 0, [partialRef, partialRef2]); const partialRef3 = makeReference( 'test:/partial.mp4', 4, 6, /* syncTime= */ null, - /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 209); + /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null); const preloadRef = makeReference( 'test:/partial.mp4', 6, 7.5, /* syncTime= */ null, - /* baseUri= */ '', /* startByte= */ 210, /* endByte= */ 419); + /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null); preloadRef.markAsPreload(); preloadRef.markAsNonIndependent(); // ref2 is not fully published yet, so it doesn't have a segment uri. const ref2 = makeReference( '', 4, 7.5, /* syncTime= */ null, - /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 419, + /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null, /* timestampOffset= */ 0, [partialRef3, preloadRef]); await testInitialManifest(master, mediaWithPartialSegments, [ref, ref2]); }); + it('parses streams with partial and preload hinted segments and BYTERANGE', async () => { // eslint-disable-line max-len + playerInterface.isLowLatencyMode = () => true; + const mediaWithPartialSegments = [ + '#EXTM3U\n', + '#EXT-X-TARGETDURATION:5\n', + '#EXT-X-PART-INF:PART-TARGET=1.5\n', + '#EXT-X-MAP:URI="init.mp4"\n', + '#EXT-X-MEDIA-SEQUENCE:0\n', + // ref includes partialRef, partialRef2 + // partialRef + '#EXT-X-PART:DURATION=2,URI="ref1.mp4",BYTERANGE=200@0,', + 'INDEPENDENT=YES\n', + // partialRef2 + '#EXT-X-PART:DURATION=2,URI="ref1.mp4",BYTERANGE=230@200,', + 'INDEPENDENT=YES\n', + '#EXTINF:4,\n', + 'ref1.mp4\n', + // ref2 includes partialRef3, preloadRef + // partialRef3 + '#EXT-X-PART:DURATION=2,URI="ref2.mp4",BYTERANGE=210@0,', + 'INDEPENDENT=YES\n', + // preloadRef + '#EXT-X-PRELOAD-HINT:TYPE=PART,URI="ref2.mp4",BYTERANGE-START=210,', + 'BYTERANGE-LENGTH=210\n', + ].join(''); + + // If ReadableStream is defined we can apply some optimizations + if (window.ReadableStream) { + const ref = makeReference( + 'test:/ref1.mp4', 0, 4, /* syncTime= */ null, + /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null, + /* timestampOffset= */ 0); + ref.markAsByterangeOptimization(); + + // ref2 is not fully published yet, so it doesn't have a segment uri. + const ref2 = makeReference( + 'test:/ref2.mp4', 4, 7.5, /* syncTime= */ null, + /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null, + /* timestampOffset= */ 0); + ref2.markAsByterangeOptimization(); + + await testInitialManifest(master, mediaWithPartialSegments, + [ref, ref2]); + } else { + const partialRef = makeReference( + 'test:/ref1.mp4', 0, 2, /* syncTime= */ null, + /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 199); + + const partialRef2 = makeReference( + 'test:/ref1.mp4', 2, 4, /* syncTime= */ null, + /* baseUri= */ '', /* startByte= */ 200, /* endByte= */ 429); + + const ref = makeReference( + 'test:/ref1.mp4', 0, 4, /* syncTime= */ null, + /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 429, + /* timestampOffset= */ 0, [partialRef, partialRef2]); + + const partialRef3 = makeReference( + 'test:/ref2.mp4', 4, 6, /* syncTime= */ null, + /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 209); + + const preloadRef = makeReference( + 'test:/ref2.mp4', 6, 7.5, /* syncTime= */ null, + /* baseUri= */ '', /* startByte= */ 210, /* endByte= */ 419); + preloadRef.markAsPreload(); + preloadRef.markAsNonIndependent(); + + // ref2 is not fully published yet, so it doesn't have a segment uri. + const ref2 = makeReference( + '', 4, 7.5, /* syncTime= */ null, + /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 419, + /* timestampOffset= */ 0, [partialRef3, preloadRef]); + + await testInitialManifest(master, mediaWithPartialSegments, + [ref, ref2]); + } + }); + // Test for https://github.com/shaka-project/shaka-player/issues/4223 it('ignores preload hinted segments without target duration', async () => { playerInterface.isLowLatencyMode = () => true; @@ -701,10 +775,9 @@ describe('HlsParser live', () => { 'main.mp4\n', // ref2 includes partialRef, but not preloadRef // partialRef - '#EXT-X-PART:DURATION=2,URI="partial.mp4",BYTERANGE=210@0,', - 'INDEPENDENT=YES\n', + '#EXT-X-PART:DURATION=2,URI="partial.mp4",INDEPENDENT=YES\n', // preloadRef - '#EXT-X-PRELOAD-HINT:TYPE=PART,URI="partial.mp4",BYTERANGE-START=210\n', + '#EXT-X-PRELOAD-HINT:TYPE=PART,URI="partial.mp4"\n', ].join(''); const ref = makeReference( @@ -714,12 +787,12 @@ describe('HlsParser live', () => { const partialRef = makeReference( 'test:/partial.mp4', 4, 6, /* syncTime= */ null, - /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 209); + /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null); // ref2 is not fully published yet, so it doesn't have a segment uri. const ref2 = makeReference( '', 4, 6, /* syncTime= */ null, - /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 209, + /* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null, /* timestampOffset= */ 0, [partialRef]); await testInitialManifest(master, mediaWithPartialSegments, [ref, ref2]); From deb2f6852610787cb8599781ee7fa00c4f737895 Mon Sep 17 00:00:00 2001 From: Alvaro Velad Galvan Date: Sat, 17 Jun 2023 18:27:54 +0200 Subject: [PATCH 2/2] Fix comments --- lib/hls/hls_parser.js | 4 ++-- lib/media/segment_reference.js | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js index 4f9da44592..b2f3729e0b 100644 --- a/lib/hls/hls_parser.js +++ b/lib/hls/hls_parser.js @@ -2967,8 +2967,8 @@ shaka.hls.HlsParser = class { } if (segmentWithByteRangeOptimization) { - // We can no optimize segments with GAPs or if the start byterange is - // different to 0 + // We cannot optimize segments with gaps, or with a start byte that is + // not 0. if (somePartialSegmentWithGap || partialSegmentRefs[0].startByte != 0) { segmentWithByteRangeOptimization = false; getUrisOptimization = null; diff --git a/lib/media/segment_reference.js b/lib/media/segment_reference.js index d7ba5b6486..ed7b5c5355 100644 --- a/lib/media/segment_reference.js +++ b/lib/media/segment_reference.js @@ -435,6 +435,9 @@ shaka.media.SegmentReference = class { /** * Mark the reference as byterange optimization. * + * The "byterange optimization" means that it is playable using MP4 low + * latency streaming with chunked data. + * * @export */ markAsByterangeOptimization() {