feat(UI): Add thumbnails to the UI (#5502)

Closes #3371

Stored content thumbnails are not supported.

demo/common/asset.js

@@ -45,6 +45,8 @@ const ShakaDemoAssetInfo = class {
     this.disabled = false;
     /** @type {!Array.<!shakaAssets.ExtraText>} */
     this.extraText = [];
+    /** @type {!Array.<string>} */
+    this.extraThumbnail = [];
     /** @type {?string} */
     this.certificateUri = null;
     /** @type {?string} */
@@ -302,6 +304,15 @@ const ShakaDemoAssetInfo = class {
     return this;
   }
 
+  /**
+   * @param {string} uri
+   * @return {!ShakaDemoAssetInfo}
+   */
+  addExtraThumbnail(uri) {
+    this.extraThumbnail.push(uri);
+    return this;
+  }
+
   /**
    * If this is called, the asset will be focused on by the integration tests.
    * @return {!ShakaDemoAssetInfo}

demo/common/assets.js

@@ -963,6 +963,16 @@ shakaAssets.testAssets = [
       .addFeature(shakaAssets.Feature.HLS)
       .addFeature(shakaAssets.Feature.MP2TS)
       .addFeature(shakaAssets.Feature.OFFLINE),
+  new ShakaDemoAssetInfo(
+      /* name= */ 'Art of Motion (DASH) (external thumbnails)',
+      /* iconUri= */ 'https://storage.googleapis.com/shaka-asset-icons/art_of_motion.png',
+      /* manifestUri= */ 'https://cdn.bitmovin.com/content/assets/art-of-motion-dash-hls-progressive/mpds/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.mpd',
+      /* source= */ shakaAssets.Source.BITCODIN)
+      .addFeature(shakaAssets.Feature.DASH)
+      .addFeature(shakaAssets.Feature.HIGH_DEFINITION)
+      .addFeature(shakaAssets.Feature.MP4)
+      .addFeature(shakaAssets.Feature.THUMBNAILS)
+      .addExtraThumbnail('https://cdn.bitmovin.com/content/assets/art-of-motion-dash-hls-progressive/thumbnails/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.vtt'),
   // End bitcodin assets }}}
 
   // MetaCDN assets {{{

demo/main.js

@@ -1358,6 +1358,10 @@ shakaDemo.Main = class {
         this.video_.poster = shakaDemo.Main.audioOnlyPoster_;
       }
 
+      for (const extraThumbnail of asset.extraThumbnail) {
+        this.player_.addThumbnailsTrack(extraThumbnail);
+      }
+
       // If the asset has an ad tag attached to it, load the ads
       const adManager = this.player_.getAdManager();
       if (adManager && asset.adTagUri) {

ui/controls.less

@@ -13,6 +13,7 @@
 @import "less/overflow_menu.less";
 @import "less/ad_controls.less";
 @import "less/tooltip.less";
+@import "less/thumbnails.less";
 @import (css, inline) "https://fonts.googleapis.com/css?family=Roboto";
 // NOTE: Working around google/material-design-icons#958
 @import (css, inline) "https://fonts.googleapis.com/icon?family=Material+Icons+Round";

ui/less/thumbnails.less

@@ -0,0 +1,26 @@
+#shaka-player-ui-thumbnail-container {
+  background-color: black;
+  border: 1px solid black;
+  box-shadow: 0 8px 8px 0 rgb(0 0 0 / 50%);
+  min-width: 150px;
+  overflow: hidden;
+  position: absolute;
+  visibility: hidden;
+  width: 15%;
+  z-index: 1;
+
+  #shaka-player-ui-thumbnail-image {
+    position: absolute;
+  }
+
+  #shaka-player-ui-thumbnail-time {
+    background-color: rgb(0 0 0 / 50%);
+    bottom: 0;
+    color: white;
+    font-size: 16px;
+    left: 0;
+    position: absolute;
+    right: 0;
+    text-align: center;
+  }
+}

ui/range_element.js

@@ -104,13 +104,28 @@ shaka.ui.RangeElement = class extends shaka.ui.Element {
       }
     });
 
+    this.eventManager.listen(this.bar, 'touchcancel', (e) => {
+      if (this.isChanging_) {
+        this.isChanging_ = false;
+        this.setBarValueForTouch_(e);
+        this.onChangeEnd();
+      }
+    });
+
     this.eventManager.listen(this.bar, 'mouseup', () => {
       if (this.isChanging_) {
         this.isChanging_ = false;
         this.onChangeEnd();
       }
     });
 
+    this.eventManager.listen(this.bar, 'blur', () => {
+      if (this.isChanging_) {
+        this.isChanging_ = false;
+        this.onChangeEnd();
+      }
+    });
+
     this.eventManager.listen(this.bar, 'contextmenu', (e) => {
       e.preventDefault();
       e.stopPropagation();

ui/seek_bar.js

@@ -83,6 +83,41 @@ shaka.ui.SeekBar = class extends shaka.ui.RangeElement {
      */
     this.wasPlaying_ = false;
 
+
+    /** @private {!HTMLElement} */
+    this.thumbnailContainer_ = shaka.util.Dom.createHTMLElement('div');
+    this.thumbnailContainer_.id = 'shaka-player-ui-thumbnail-container';
+
+    /** @private {!HTMLImageElement} */
+    this.thumbnailImage_ = /** @type {!HTMLImageElement} */ (
+      shaka.util.Dom.createHTMLElement('img'));
+    this.thumbnailImage_.id = 'shaka-player-ui-thumbnail-image';
+    this.thumbnailImage_.draggable = false;
+
+    /** @private {!HTMLElement} */
+    this.thumbnailTime_ = shaka.util.Dom.createHTMLElement('div');
+    this.thumbnailTime_.id = 'shaka-player-ui-thumbnail-time';
+
+    this.thumbnailContainer_.appendChild(this.thumbnailImage_);
+    this.thumbnailContainer_.appendChild(this.thumbnailTime_);
+    this.container.appendChild(this.thumbnailContainer_);
+
+    /**
+     * True if the bar is moving due to touchscreen or keyboard events.
+     *
+     * @private {boolean}
+     */
+    this.isMoving_ = false;
+
+    /**
+     * The timer is activated to hide the thumbnail.
+     *
+     * @private {shaka.util.Timer}
+     */
+    this.hideThumbnailTimer_ = new shaka.util.Timer(() => {
+      this.hideThumbnail_();
+    });
+
     /** @private {!Array.<!shaka.extern.AdCuePoint>} */
     this.adCuePoints_ = [];
 
@@ -120,6 +155,29 @@ shaka.ui.SeekBar = class extends shaka.ui.RangeElement {
           this.onAdCuePointsChanged_();
         });
 
+    this.eventManager.listen(this.bar, 'mousemove', (event) => {
+      if (!this.player.getImageTracks().length) {
+        this.hideThumbnail_();
+        return;
+      }
+      const rect = this.bar.getBoundingClientRect();
+      const min = parseFloat(this.bar.min);
+      const max = parseFloat(this.bar.max);
+      // Pixels from the left of the range element
+      const mousePosition = event.clientX - rect.left;
+      // Pixels per unit value of the range element.
+      const scale = (max - min) / rect.width;
+      // Mouse position in units, which may be outside the allowed range.
+      const value = Math.round(min + scale * mousePosition);
+      // Show Thumbnail
+      this.showThumbnail_(mousePosition, value);
+    });
+
+    this.eventManager.listen(this.container, 'mouseleave', () => {
+      this.hideThumbnailTimer_.stop();
+      this.hideThumbnailTimer_.tickAfter(/* seconds= */ 0.25);
+    });
+
     // Initialize seek state and label.
     this.setValue(this.video.currentTime);
     this.update();
@@ -153,6 +211,8 @@ shaka.ui.SeekBar = class extends shaka.ui.RangeElement {
     this.wasPlaying_ = !this.video.paused;
     this.controls.setSeeking(true);
     this.video.pause();
+    this.hideThumbnailTimer_.stop();
+    this.isMoving_ = true;
   }
 
   /**
@@ -180,6 +240,18 @@ shaka.ui.SeekBar = class extends shaka.ui.RangeElement {
     // Calling |start| on an already pending timer will cancel the old request
     // and start the new one.
     this.seekTimer_.tickAfter(/* seconds= */ 0.125);
+
+    if (this.player.getImageTracks().length) {
+      const min = parseFloat(this.bar.min);
+      const max = parseFloat(this.bar.max);
+      const rect = this.bar.getBoundingClientRect();
+      const value = Math.round(this.getValue());
+      const scale = (max - min) / rect.width;
+      const position = (value - min) / scale;
+      this.showThumbnail_(position, value);
+    } else {
+      this.hideThumbnail_();
+    }
   }
 
   /**
@@ -197,6 +269,12 @@ shaka.ui.SeekBar = class extends shaka.ui.RangeElement {
     if (this.wasPlaying_) {
       this.video.play();
     }
+
+    if (this.isMoving_) {
+      this.isMoving_ = false;
+      this.hideThumbnailTimer_.stop();
+      this.hideThumbnailTimer_.tickAfter(/* seconds= */ 0.25);
+    }
   }
 
   /**
@@ -376,6 +454,151 @@ shaka.ui.SeekBar = class extends shaka.ui.RangeElement {
   updateAriaLabel_() {
     this.bar.ariaLabel = this.localization.resolve(shaka.ui.Locales.Ids.SEEK);
   }
+
+
+  /**
+   * @private
+   */
+  async showThumbnail_(pixelPosition, value) {
+    const thumbnailTrack = this.getThumbnailTrack_();
+    if (!thumbnailTrack) {
+      this.hideThumbnail_();
+      return;
+    }
+    if (value < 0) {
+      value = 0;
+    }
+    const seekRange = this.player.seekRange();
+    const playerValue = Math.max(Math.ceil(seekRange.start),
+        Math.min(Math.floor(seekRange.end), value));
+    const thumbnail =
+        await this.player.getThumbnails(thumbnailTrack.id, playerValue);
+    if (!thumbnail || !thumbnail.uris.length) {
+      this.hideThumbnail_();
+      return;
+    }
+    if (this.player.isLive()) {
+      const totalSeconds = seekRange.end - value;
+      if (totalSeconds < 1) {
+        this.thumbnailTime_.textContent =
+            this.localization.resolve(shaka.ui.Locales.Ids.LIVE);
+      } else {
+        this.thumbnailTime_.textContent =
+            '-' + this.timeFormatter_(totalSeconds);
+      }
+    } else {
+      this.thumbnailTime_.textContent = this.timeFormatter_(value);
+    }
+    const offsetTop = -10;
+    const width = this.thumbnailContainer_.clientWidth;
+    let height = Math.floor(width * 9 / 16);
+    this.thumbnailContainer_.style.height = height + 'px';
+    this.thumbnailContainer_.style.top = -(height - offsetTop) + 'px';
+    const leftPosition = Math.min(this.bar.offsetWidth - width,
+        Math.max(0, pixelPosition - (width / 2)));
+    this.thumbnailContainer_.style.left = leftPosition + 'px';
+    this.thumbnailContainer_.style.visibility = 'visible';
+    const uri = thumbnail.uris[0].split('#xywh=')[0];
+    if (uri !== this.thumbnailImage_.src) {
+      try {
+        this.thumbnailContainer_.removeChild(this.thumbnailImage_);
+      } catch (e) {
+        // The image is not a child
+      }
+      this.thumbnailImage_ = /** @type {!HTMLImageElement} */ (
+        shaka.util.Dom.createHTMLElement('img'));
+      this.thumbnailImage_.id = 'shaka-player-ui-thumbnail-image';
+      this.thumbnailImage_.draggable = false;
+      this.thumbnailImage_.src = uri;
+      this.thumbnailContainer_.insertBefore(this.thumbnailImage_,
+          this.thumbnailContainer_.firstChild);
+    }
+    const scale = width / thumbnail.width;
+    if (thumbnail.imageHeight) {
+      this.thumbnailImage_.height = thumbnail.imageHeight;
+    } else if (!thumbnail.sprite) {
+      this.thumbnailImage_.style.height = '100%';
+      this.thumbnailImage_.style.objectFit = 'contain';
+    }
+    if (thumbnail.imageWidth) {
+      this.thumbnailImage_.width = thumbnail.imageWidth;
+    } else if (!thumbnail.sprite) {
+      this.thumbnailImage_.style.width = '100%';
+      this.thumbnailImage_.style.objectFit = 'contain';
+    }
+    this.thumbnailImage_.style.left = '-' + scale * thumbnail.positionX + 'px';
+    this.thumbnailImage_.style.top = '-' + scale * thumbnail.positionY + 'px';
+    this.thumbnailImage_.style.transform = 'scale(' + scale + ')';
+    this.thumbnailImage_.style.transformOrigin = 'left top';
+    // Update container height and top
+    height = Math.floor(width * thumbnail.height / thumbnail.width);
+    this.thumbnailContainer_.style.height = height + 'px';
+    this.thumbnailContainer_.style.top = -(height - offsetTop) + 'px';
+  }
+
+
+  /**
+   * @return {?shaka.extern.Track} The thumbnail track.
+   * @private
+   */
+  getThumbnailTrack_() {
+    const imageTracks = this.player.getImageTracks();
+    if (!imageTracks.length) {
+      return null;
+    }
+    const mimeTypesPreference = [
+      'image/avif',
+      'image/webp',
+      'image/jpeg',
+      'image/png',
+      'image/svg+xml',
+    ];
+    for (const mimeType of mimeTypesPreference) {
+      const estimatedBandwidth = this.player.getStats().estimatedBandwidth;
+      const bestOptions = imageTracks.filter((track) => {
+        return track.mimeType.toLowerCase() === mimeType &&
+            track.bandwidth < estimatedBandwidth * 0.01;
+      }).sort((a, b) => {
+        return b.bandwidth - a.bandwidth;
+      });
+      if (bestOptions && bestOptions.length) {
+        return bestOptions[0];
+      }
+    }
+    return imageTracks[0];
+  }
+
+
+  /**
+   * @private
+   */
+  hideThumbnail_() {
+    this.thumbnailContainer_.style.visibility = 'hidden';
+    this.thumbnailTime_.textContent = '';
+  }
+
+
+  /**
+   * @param {number} totalSeconds
+   * @private
+   */
+  timeFormatter_(totalSeconds) {
+    const secondsNumber = Math.round(totalSeconds);
+    const hours = Math.floor(secondsNumber / 3600);
+    let minutes = Math.floor((secondsNumber - (hours * 3600)) / 60);
+    let seconds = secondsNumber - (hours * 3600) - (minutes * 60);
+    if (seconds < 10) {
+      seconds = '0' + seconds;
+    }
+    if (hours > 0) {
+      if (minutes < 10) {
+        minutes = '0' + minutes;
+      }
+      return hours + ':' + minutes + ':' + seconds;
+    } else {
+      return minutes + ':' + seconds;
+    }
+  }
 };