Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(HLS): Optimization of LL-HLS with byterange #5319

Merged
merged 3 commits into from
Jun 21, 2023
Merged

feat(HLS): Optimization of LL-HLS with byterange #5319

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b43b39f
feat(HLS): Optimization of LL-HLS with byterange
avelad Jun 16, 2023
deb2f68
Fix comments
avelad Jun 17, 2023
024b981
Merge branch 'main' into ll-hls-byterange-optimization
avelad Jun 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 48 additions & 4 deletions lib/hls/hls_parser.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2781,14 +2781,15 @@ shaka.hls.HlsParser = class {
* @param {!Map.<string, string>} variables
* @param {string} absoluteMediaPlaylistUri
* @param {string} type
* @param {string} mimeType
* @param {number} timestampOffset
* @param {shaka.extern.HlsAes128Key=} hlsAes128Key
* @return {shaka.media.SegmentReference}
* @private
*/
createSegmentReference_(
initSegmentReference, previousReference, hlsSegment, startTime,
variables, absoluteMediaPlaylistUri, type, timestampOffset,
variables, absoluteMediaPlaylistUri, type, mimeType, timestampOffset,
hlsAes128Key) {
const tags = hlsSegment.tags;
const extinfTag =
Expand Down Expand Up @@ -2836,8 +2837,18 @@ shaka.hls.HlsParser = class {
}

// Create SegmentReferences for the partial segments.
const partialSegmentRefs = [];
if (this.lowLatencyMode_) {
let partialSegmentRefs = [];

// Optimization for LL-HLS with byterange
// More info in https://tinyurl.com/hls-open-byte-range
let segmentWithByteRangeOptimization = false;
let getUrisOptimization = null;
let somePartialSegmentWithGap = false;

if (this.lowLatencyMode_ && hlsSegment.partialSegments.length) {
const byterangeOptimizationSupport = (mimeType == 'video/mp4' ||
mimeType == 'audio/mp4') && window.ReadableStream;

let partialSyncTime = syncTime;
for (let i = 0; i < hlsSegment.partialSegments.length; i++) {
const item = hlsSegment.partialSegments[i];
Expand Down Expand Up @@ -2870,6 +2881,11 @@ shaka.hls.HlsParser = class {
const pByterangeLength = item.getAttributeValue('BYTERANGE-LENGTH');
if (pByterangeLength) {
pEndByte = pStartByte + Number(pByterangeLength) - 1;
} else if (pStartByte) {
// If we have a non-zero start byte, but no end byte, follow the
// recommendation of https://tinyurl.com/hls-open-byte-range and
// set the end byte explicitly to a large integer.
pEndByte = Number.MAX_SAFE_INTEGER;
}
} else {
const pByterange = item.getAttributeValue('BYTERANGE');
Expand All @@ -2884,6 +2900,7 @@ shaka.hls.HlsParser = class {
let partialStatus = shaka.media.SegmentReference.Status.AVAILABLE;
if (item.getAttributeValue('GAP') == 'YES') {
partialStatus = shaka.media.SegmentReference.Status.MISSING;
somePartialSegmentWithGap = true;
}

let pAbsoluteUri = null;
Expand All @@ -2896,6 +2913,12 @@ shaka.hls.HlsParser = class {
return [pAbsoluteUri];
};

if (byterangeOptimizationSupport &&
pStartByte >= 0 && pEndByte != null) {
getUrisOptimization = getPartialUris;
segmentWithByteRangeOptimization = true;
}

const partial = new shaka.media.SegmentReference(
pStartTime,
pEndTime,
Expand Down Expand Up @@ -2947,6 +2970,17 @@ shaka.hls.HlsParser = class {
endTime = partialSegmentRefs[partialSegmentRefs.length - 1].endTime;
}

if (segmentWithByteRangeOptimization) {
// We cannot optimize segments with gaps, or with a start byte that is
// not 0.
if (somePartialSegmentWithGap || partialSegmentRefs[0].startByte != 0) {
segmentWithByteRangeOptimization = false;
getUrisOptimization = null;
} else {
partialSegmentRefs = [];
}
}

// If the segment has EXT-X-BYTERANGE tag, set the start byte and end byte
// base on the byterange information. If segment has no EXT-X-BYTERANGE tag
// and has partial segments, set the start byte and end byte base on the
Expand Down Expand Up @@ -2979,14 +3013,17 @@ shaka.hls.HlsParser = class {

let absoluteSegmentUri = null;
const getUris = () => {
if (getUrisOptimization) {
return getUrisOptimization();
}
if (absoluteSegmentUri == null) {
absoluteSegmentUri = this.variableSubstitution_(
hlsSegment.absoluteUri, variables);
}
return absoluteSegmentUri.length ? [absoluteSegmentUri] : [];
};

return new shaka.media.SegmentReference(
const reference = new shaka.media.SegmentReference(
startTime,
endTime,
getUris,
Expand All @@ -3003,6 +3040,12 @@ shaka.hls.HlsParser = class {
status,
hlsAes128Key,
);

if (segmentWithByteRangeOptimization) {
reference.markAsByterangeOptimization();
}

return reference;
}


Expand Down Expand Up @@ -3129,6 +3172,7 @@ shaka.hls.HlsParser = class {
variables,
playlist.absoluteUri,
type,
mimeType,
lastDiscontinuityStartTime,
hlsAes128Key);
previousReference = reference;
Expand Down
10 changes: 1 addition & 9 deletions lib/hls/manifest_text_parser.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,15 +158,7 @@ shaka.hls.ManifestTextParser = class {
partialSegmentTags.push(tag);
} else if (tag.name == 'EXT-X-PRELOAD-HINT') {
if (tag.getAttributeValue('TYPE') == 'PART') {
// Note: BYTERANGE-START without BYTERANGE-LENGTH is being
// ignored.
if (tag.getAttributeValue('BYTERANGE-START') != null) {
if (tag.getAttributeValue('BYTERANGE-LENGTH') != null) {
partialSegmentTags.push(tag);
}
} else {
partialSegmentTags.push(tag);
}
partialSegmentTags.push(tag);
} else if (tag.getAttributeValue('TYPE') == 'MAP') {
// Rename the Preload Hint tag to be a Map tag.
tag.setName('EXT-X-MAP');
Expand Down
8 changes: 7 additions & 1 deletion lib/media/segment_prefetch.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,13 @@ shaka.media.SegmentPrefetch = class {
let reference = startReference;
while (this.segmentPrefetchMap_.size < this.prefetchLimit_ &&
reference != null) {
if (!this.segmentPrefetchMap_.has(reference)) {
// By default doesn't prefech preload partial segments when using
// byterange
let prefetchAllowed = true;
if (reference.isPreload() && reference.endByte != null) {
prefetchAllowed = false;
}
if (prefetchAllowed && !this.segmentPrefetchMap_.has(reference)) {
const segmentPrefetchOperation =
new shaka.media.SegmentPrefetchOperation(this.fetchDispatcher_);
segmentPrefetchOperation.dispatchFetch(reference, this.stream_);
Expand Down
25 changes: 25 additions & 0 deletions lib/media/segment_reference.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,9 @@ shaka.media.SegmentReference = class {
/** @type {boolean} */
this.independent = true;

/** @type {boolean} */
this.byterangeOptimization = false;

/** @type {?shaka.extern.HlsAes128Key} */
this.hlsAes128Key = hlsAes128Key;

Expand Down Expand Up @@ -429,6 +432,28 @@ shaka.media.SegmentReference = class {
return this.independent;
}

/**
* Mark the reference as byterange optimization.
avelad marked this conversation as resolved.
Show resolved Hide resolved
*
* The "byterange optimization" means that it is playable using MP4 low
* latency streaming with chunked data.
*
* @export
*/
markAsByterangeOptimization() {
this.byterangeOptimization = true;
}

/**
* Returns true if the segment has a byterange optimization.
*
* @return {boolean}
* @export
*/
hasByterangeOptimization() {
return this.byterangeOptimization;
}

/**
* Set the segment's thumbnail sprite.
*
Expand Down
5 changes: 3 additions & 2 deletions lib/media/streaming_engine.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1330,9 +1330,10 @@ shaka.media.StreamingEngine = class {
stream.mimeType == 'audio/mp4';
const isReadableStreamSupported = window.ReadableStream;
// Enable MP4 low latency streaming with ReadableStream chunked data.
// And only for DASH.
// And only for DASH and HLS with byterange optimization.
if (this.config_.lowLatencyMode && isReadableStreamSupported && isMP4 &&
this.manifest_.type != shaka.media.ManifestParser.HLS) {
(this.manifest_.type != shaka.media.ManifestParser.HLS ||
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Say, it occurs to me. This assumes that a manifest that isn't HLS is DASH, based on the comment above.
But we now have the constant of shaka.media.ManifestParser.MSS. Is this behavior supported by MSS?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, this would work for MSS live (in the future)

reference.hasByterangeOptimization())) {
let remaining = new Uint8Array(0);
let processingResult = false;
let callbackCalled = false;
Expand Down
113 changes: 93 additions & 20 deletions test/hls/hls_live_unit.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -634,58 +634,132 @@ describe('HlsParser live', () => {
'#EXTM3U\n',
'#EXT-X-TARGETDURATION:5\n',
'#EXT-X-PART-INF:PART-TARGET=1.5\n',
'#EXT-X-MAP:URI="init.mp4",BYTERANGE="616@0"\n',
'#EXT-X-MAP:URI="init.mp4"\n',
'#EXT-X-MEDIA-SEQUENCE:0\n',
// ref includes partialRef, partialRef2
// partialRef
'#EXT-X-PART:DURATION=2,URI="partial.mp4",BYTERANGE=200@0,',
'INDEPENDENT=YES\n',
'#EXT-X-PART:DURATION=2,URI="partial.mp4",INDEPENDENT=YES\n',
// partialRef2
'#EXT-X-PART:DURATION=2,URI="partial2.mp4",BYTERANGE=230@200,',
'INDEPENDENT=YES\n',
'#EXT-X-PART:DURATION=2,URI="partial2.mp4",INDEPENDENT=YES\n',
'#EXTINF:4,\n',
'main.mp4\n',
// ref2 includes partialRef3, preloadRef
// partialRef3
'#EXT-X-PART:DURATION=2,URI="partial.mp4",BYTERANGE=210@0,',
'INDEPENDENT=YES\n',
'#EXT-X-PART:DURATION=2,URI="partial.mp4",INDEPENDENT=YES\n',
// preloadRef
'#EXT-X-PRELOAD-HINT:TYPE=PART,URI="partial.mp4",BYTERANGE-START=210,',
'BYTERANGE-LENGTH=210\n',
'#EXT-X-PRELOAD-HINT:TYPE=PART,URI="partial.mp4"\n',
].join('');

const partialRef = makeReference(
'test:/partial.mp4', 0, 2, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 199);
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null);

const partialRef2 = makeReference(
'test:/partial2.mp4', 2, 4, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 200, /* endByte= */ 429);
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null);

const ref = makeReference(
'test:/main.mp4', 0, 4, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 429,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null,
/* timestampOffset= */ 0, [partialRef, partialRef2]);

const partialRef3 = makeReference(
'test:/partial.mp4', 4, 6, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 209);
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null);

const preloadRef = makeReference(
'test:/partial.mp4', 6, 7.5, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 210, /* endByte= */ 419);
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null);
preloadRef.markAsPreload();
preloadRef.markAsNonIndependent();

// ref2 is not fully published yet, so it doesn't have a segment uri.
const ref2 = makeReference(
'', 4, 7.5, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 419,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null,
/* timestampOffset= */ 0, [partialRef3, preloadRef]);

await testInitialManifest(master, mediaWithPartialSegments, [ref, ref2]);
});

it('parses streams with partial and preload hinted segments and BYTERANGE', async () => { // eslint-disable-line max-len
playerInterface.isLowLatencyMode = () => true;
const mediaWithPartialSegments = [
'#EXTM3U\n',
'#EXT-X-TARGETDURATION:5\n',
'#EXT-X-PART-INF:PART-TARGET=1.5\n',
'#EXT-X-MAP:URI="init.mp4"\n',
'#EXT-X-MEDIA-SEQUENCE:0\n',
// ref includes partialRef, partialRef2
// partialRef
'#EXT-X-PART:DURATION=2,URI="ref1.mp4",BYTERANGE=200@0,',
'INDEPENDENT=YES\n',
// partialRef2
'#EXT-X-PART:DURATION=2,URI="ref1.mp4",BYTERANGE=230@200,',
'INDEPENDENT=YES\n',
'#EXTINF:4,\n',
'ref1.mp4\n',
// ref2 includes partialRef3, preloadRef
// partialRef3
'#EXT-X-PART:DURATION=2,URI="ref2.mp4",BYTERANGE=210@0,',
'INDEPENDENT=YES\n',
// preloadRef
'#EXT-X-PRELOAD-HINT:TYPE=PART,URI="ref2.mp4",BYTERANGE-START=210,',
'BYTERANGE-LENGTH=210\n',
].join('');

// If ReadableStream is defined we can apply some optimizations
if (window.ReadableStream) {
const ref = makeReference(
'test:/ref1.mp4', 0, 4, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null,
/* timestampOffset= */ 0);
ref.markAsByterangeOptimization();

// ref2 is not fully published yet, so it doesn't have a segment uri.
const ref2 = makeReference(
'test:/ref2.mp4', 4, 7.5, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null,
/* timestampOffset= */ 0);
ref2.markAsByterangeOptimization();

await testInitialManifest(master, mediaWithPartialSegments,
[ref, ref2]);
} else {
const partialRef = makeReference(
'test:/ref1.mp4', 0, 2, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 199);

const partialRef2 = makeReference(
'test:/ref1.mp4', 2, 4, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 200, /* endByte= */ 429);

const ref = makeReference(
'test:/ref1.mp4', 0, 4, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 429,
/* timestampOffset= */ 0, [partialRef, partialRef2]);

const partialRef3 = makeReference(
'test:/ref2.mp4', 4, 6, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 209);

const preloadRef = makeReference(
'test:/ref2.mp4', 6, 7.5, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 210, /* endByte= */ 419);
preloadRef.markAsPreload();
preloadRef.markAsNonIndependent();

// ref2 is not fully published yet, so it doesn't have a segment uri.
const ref2 = makeReference(
'', 4, 7.5, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 419,
/* timestampOffset= */ 0, [partialRef3, preloadRef]);

await testInitialManifest(master, mediaWithPartialSegments,
[ref, ref2]);
}
});

// Test for https://github.com/shaka-project/shaka-player/issues/4223
it('ignores preload hinted segments without target duration', async () => {
playerInterface.isLowLatencyMode = () => true;
Expand All @@ -701,10 +775,9 @@ describe('HlsParser live', () => {
'main.mp4\n',
// ref2 includes partialRef, but not preloadRef
// partialRef
'#EXT-X-PART:DURATION=2,URI="partial.mp4",BYTERANGE=210@0,',
'INDEPENDENT=YES\n',
'#EXT-X-PART:DURATION=2,URI="partial.mp4",INDEPENDENT=YES\n',
// preloadRef
'#EXT-X-PRELOAD-HINT:TYPE=PART,URI="partial.mp4",BYTERANGE-START=210\n',
'#EXT-X-PRELOAD-HINT:TYPE=PART,URI="partial.mp4"\n',
].join('');

const ref = makeReference(
Expand All @@ -714,12 +787,12 @@ describe('HlsParser live', () => {

const partialRef = makeReference(
'test:/partial.mp4', 4, 6, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 209);
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null);

// ref2 is not fully published yet, so it doesn't have a segment uri.
const ref2 = makeReference(
'', 4, 6, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 209,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null,
/* timestampOffset= */ 0, [partialRef]);

await testInitialManifest(master, mediaWithPartialSegments, [ref, ref2]);
Expand Down