refactor: Move caption parser to webworker, saving 5732b offloading w…

…ork (#863)

package.json

@@ -62,7 +62,7 @@
     "global": "^4.3.2",
     "m3u8-parser": "4.4.2",
     "mpd-parser": "0.10.1",
-    "mux.js": "5.6.2",
+    "mux.js": "5.6.5",
     "video.js": "^6 || ^7"
   },
   "devDependencies": {

src/dash-playlist-loader.js

@@ -8,7 +8,7 @@ import {
   updateMaster as updatePlaylist
 } from './playlist-loader';
 import { resolveUrl, resolveManifestRedirect } from './resolve-url';
-import mp4Inspector from 'mux.js/lib/tools/mp4-inspector';
+import parseSidx from 'mux.js/lib/tools/parse-sidx';
 import { segmentXhrHeaders } from './xhr';
 import window from 'global/window';
 import {
@@ -345,7 +345,7 @@ export default class DashPlaylistLoader extends EventTarget {
       }
 
       const bytes = toUint8(request.response);
-      const sidx = mp4Inspector.parseSidx(bytes.subarray(8));
+      const sidx = parseSidx(bytes.subarray(8));
 
       return doneFn(master, sidx);
     };

src/media-segment-request.js

@@ -148,7 +148,7 @@ const handleKeyResponse = (segment, finishProcessingFn) => (error, request) => {
  *                                        this request
  */
 const handleInitSegmentResponse =
-({segment, captionParser, finishProcessingFn}) => (error, request) => {
+({segment, finishProcessingFn}) => (error, request) => {
   const response = request.response;
   const errorObj = handleErrors(error, request);
 
@@ -368,7 +368,6 @@ const handleSegmentBytes = ({
   segment,
   bytes,
   isPartial,
-  captionParser,
   trackInfoFn,
   timingInfoFn,
   videoSegmentTimingInfoFn,
@@ -430,30 +429,52 @@ const handleSegmentBytes = ({
       timingInfoFn(segment, 'video', 'start', timingInfo);
     }
 
-    // if the track still has audio at this point it is only possible
-    // for it to be audio only. See `tracks.video && tracks.audio` if statement
-    // above.
-    dataFn(segment, {data: bytes, type: trackInfo.hasAudio ? 'audio' : 'video'});
+    const finishLoading = (captions) => {
+      // if the track still has audio at this point it is only possible
+      // for it to be audio only. See `tracks.video && tracks.audio` if statement
+      // above.
+      // we make sure to use segment.bytes here as that
+      dataFn(segment, {data: bytes, type: trackInfo.hasAudio ? 'audio' : 'video'});
+      if (captions && captions.length) {
+        captionsFn(segment, captions);
+      }
+      doneFn(null, segment, {});
+    };
 
     // Run through the CaptionParser in case there are captions.
     // Initialize CaptionParser if it hasn't been yet
-    if (captionParser && tracks.video) {
-      if (!captionParser.isInitialized()) {
-        captionParser.init();
+    if (!tracks.video || !bytes.byteLength || !segment.transmuxer) {
+      finishLoading();
+      return;
+    }
+
+    const buffer = bytes instanceof ArrayBuffer ? bytes : bytes.buffer;
+    const byteOffset = bytes instanceof ArrayBuffer ? 0 : bytes.byteOffset;
+    const listenForCaptions = (event) => {
+      if (event.data.action !== 'mp4Captions') {
+        return;
       }
+      segment.transmuxer.removeEventListener('message', listenForCaptions);
 
-      const parsed = captionParser.parse(
-        segment.bytes,
-        [tracks.video.id],
-        segment.map.timescales
-      );
+      const data = event.data.data;
 
-      if (parsed && parsed.captions && parsed.captions.length > 0) {
-        captionsFn(segment, parsed.captions);
-      }
-    }
+      // transfer ownership of bytes back to us.
+      segment.bytes = bytes = new Uint8Array(data, data.byteOffset || 0, data.byteLength);
 
-    doneFn(null, segment, {});
+      finishLoading(event.data.captions);
+    };
+
+    segment.transmuxer.addEventListener('message', listenForCaptions);
+
+    // transfer ownership of bytes to worker.
+    segment.transmuxer.postMessage({
+      action: 'pushMp4Captions',
+      timescales: segment.map.timescales,
+      trackIds: [tracks.video.id],
+      data: buffer,
+      byteOffset,
+      byteLength: bytes.byteLength
+    }, [ buffer ]);
     return;
   }
 
@@ -493,7 +514,6 @@ const handleSegmentBytes = ({
 const decryptSegment = ({
   decryptionWorker,
   segment,
-  captionParser,
   trackInfoFn,
   timingInfoFn,
   videoSegmentTimingInfoFn,
@@ -517,7 +537,6 @@ const decryptSegment = ({
         segment,
         bytes: segment.bytes,
         isPartial: false,
-        captionParser,
         trackInfoFn,
         timingInfoFn,
         videoSegmentTimingInfoFn,
@@ -573,7 +592,6 @@ const decryptSegment = ({
 const waitForCompletion = ({
   activeXhrs,
   decryptionWorker,
-  captionParser,
   trackInfoFn,
   timingInfoFn,
   videoSegmentTimingInfoFn,
@@ -619,7 +637,6 @@ const waitForCompletion = ({
         return decryptSegment({
           decryptionWorker,
           segment,
-          captionParser,
           trackInfoFn,
           timingInfoFn,
           videoSegmentTimingInfoFn,
@@ -634,7 +651,6 @@ const waitForCompletion = ({
         segment,
         bytes: segment.bytes,
         isPartial: false,
-        captionParser,
         trackInfoFn,
         timingInfoFn,
         videoSegmentTimingInfoFn,
@@ -795,7 +811,6 @@ export const mediaSegmentRequest = ({
   xhr,
   xhrOptions,
   decryptionWorker,
-  captionParser,
   segment,
   abortFn,
   progressFn,
@@ -812,7 +827,6 @@ export const mediaSegmentRequest = ({
   const finishProcessingFn = waitForCompletion({
     activeXhrs,
     decryptionWorker,
-    captionParser,
     trackInfoFn,
     timingInfoFn,
     videoSegmentTimingInfoFn,
@@ -843,7 +857,6 @@ export const mediaSegmentRequest = ({
     });
     const initSegmentRequestCallback = handleInitSegmentResponse({
       segment,
-      captionParser,
       finishProcessingFn
     });
     const initSegmentXhr = xhr(initSegmentOptions, initSegmentRequestCallback);

src/segment-loader.js

@@ -11,7 +11,6 @@ import TransmuxWorker from 'worker!./transmuxer-worker.worker.js';
 import segmentTransmuxer from './segment-transmuxer';
 import { TIME_FUDGE_FACTOR, timeUntilRebuffer as timeUntilRebuffer_ } from './ranges';
 import { minRebufferMaxBandwidthSelector } from './playlist-selectors';
-import CaptionParser from 'mux.js/lib/mp4/caption-parser';
 import logger from './util/logger';
 import { concatSegments } from './util/segment';
 import {
@@ -397,13 +396,6 @@ export default class SegmentLoader extends videojs.EventTarget {
     this.cacheEncryptionKeys_ = settings.cacheEncryptionKeys;
     this.keyCache_ = {};
 
-    // Fmp4 CaptionParser
-    if (this.loaderType_ === 'main') {
-      this.captionParser_ = new CaptionParser();
-    } else {
-      this.captionParser_ = null;
-    }
-
     this.decrypter_ = settings.decrypter;
 
     // Manages the tracking and generation of sync-points, mappings
@@ -522,9 +514,6 @@ export default class SegmentLoader extends videojs.EventTarget {
       segmentTransmuxer.dispose();
     }
     this.resetStats_();
-    if (this.captionParser_) {
-      this.captionParser_.reset();
-    }
 
     if (this.checkBufferTimeout_) {
       window.clearTimeout(this.checkBufferTimeout_);
@@ -928,8 +917,10 @@ export default class SegmentLoader extends videojs.EventTarget {
     this.remove(0, Infinity, done);
 
     // clears fmp4 captions
-    if (this.captionParser_) {
-      this.captionParser_.clearAllCaptions();
+    if (this.transmuxer_) {
+      this.transmuxer_.postMessage({
+        action: 'clearAllMp4Captions'
+      });
     }
   }
 
@@ -962,8 +953,10 @@ export default class SegmentLoader extends videojs.EventTarget {
     this.metadataQueue_.caption = [];
     this.abort();
 
-    if (this.captionParser_) {
-      this.captionParser_.clearParsedCaptions();
+    if (this.transmuxer_) {
+      this.transmuxer_.postMessage({
+        action: 'clearParsedMp4Captions'
+      });
     }
   }
 
@@ -1547,8 +1540,11 @@ export default class SegmentLoader extends videojs.EventTarget {
 
     // Reset stored captions since we added parsed
     // captions to a text track at this point
-    if (this.captionParser_) {
-      this.captionParser_.clearParsedCaptions();
+
+    if (this.transmuxer_) {
+      this.transmuxer_.postMessage({
+        action: 'clearParsedMp4Captions'
+      });
     }
   }
 
@@ -1958,8 +1954,10 @@ export default class SegmentLoader extends videojs.EventTarget {
     this.trimBackBuffer_(segmentInfo);
 
     if (typeof segmentInfo.timestampOffset === 'number') {
-      if (this.captionParser_) {
-        this.captionParser_.clearAllCaptions();
+      if (this.transmuxer_) {
+        this.transmuxer_.postMessage({
+          action: 'clearAllMp4Captions'
+        });
       }
     }
 
@@ -2035,7 +2033,6 @@ export default class SegmentLoader extends videojs.EventTarget {
       xhr: this.vhs_.xhr,
       xhrOptions: this.xhrOptions_,
       decryptionWorker: this.decrypter_,
-      captionParser: this.captionParser_,
       segment: simpleSegment,
       handlePartialData: this.handlePartialData_,
       abortFn: this.handleAbort_.bind(this),

src/transmuxer-worker.js

@@ -16,6 +16,7 @@
 
 import {Transmuxer as FullMux} from 'mux.js/lib/mp4/transmuxer';
 import PartialMux from 'mux.js/lib/partial/transmuxer';
+import CaptionParser from 'mux.js/lib/mp4/caption-parser';
 import {
   secondsToVideoTs,
   videoTsToSeconds
@@ -134,6 +135,7 @@ const wireFullTransmuxerEvents = function(self, transmuxer) {
       }
     });
   });
+
 };
 
 const wirePartialTransmuxerEvents = function(self, transmuxer) {
@@ -287,6 +289,38 @@ class MessageHandlers {
     } else {
       wireFullTransmuxerEvents(this.self, this.transmuxer);
     }
+
+  }
+
+  pushMp4Captions(data) {
+    if (!this.captionParser) {
+      this.captionParser = new CaptionParser();
+      this.captionParser.init();
+    }
+    const segment = new Uint8Array(data.data, data.byteOffset, data.byteLength);
+    const parsed = this.captionParser.parse(
+      segment,
+      data.trackIds,
+      data.timescales
+    );
+
+    this.self.postMessage({
+      action: 'mp4Captions',
+      captions: parsed.captions,
+      data: segment.buffer
+    }, [segment.buffer]);
+  }
+
+  clearAllMp4Captions() {
+    if (this.captionParser) {
+      this.captionParser.clearAllCaptions();
+    }
+  }
+
+  clearParsedMp4Captions() {
+    if (this.captionParser) {
+      this.captionParser.clearParsedCaptions();
+    }
   }
 
   /**