diff --git a/build/types/core b/build/types/core
index bd302657de..7682b8800c 100644
--- a/build/types/core
+++ b/build/types/core
@@ -23,6 +23,7 @@
 +../../lib/media/media_source_engine.js
 +../../lib/media/mp4_segment_index_parser.js
 +../../lib/media/period_observer.js
++../../lib/media/play_rate_controller.js
 +../../lib/media/playhead.js
 +../../lib/media/playhead_observer.js
 +../../lib/media/presentation_timeline.js
diff --git a/lib/media/play_rate_controller.js b/lib/media/play_rate_controller.js
new file mode 100644
index 0000000000..eaf9d4df8a
--- /dev/null
+++ b/lib/media/play_rate_controller.js
@@ -0,0 +1,208 @@
+/**
+ * @license
+ * Copyright 2016 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+goog.provide('shaka.media.PlayRateController');
+
+goog.require('shaka.util.IReleasable');
+goog.require('shaka.util.Timer');
+
+/**
+ * The play rate controller controls the playback rate on the media element.
+ * This provides some missing functionality (e.g. negative playback rate). If
+ * the playback rate on the media element can change outside of the controller,
+ * the playback controller will need to be updated to stay in-sync.
+ *
+ * @implements {shaka.util.IReleasable}
+ * @final
+ */
+shaka.media.PlayRateController = class {
+  /**
+   * @param {shaka.media.PlayRateController.Harness} harness
+   */
+  constructor(harness) {
+    /** @private {?shaka.media.PlayRateController.Harness} */
+    this.harness_ = harness;
+
+    /** @private {!Set.<shaka.media.PlayRateController.Modifier>} */
+    this.modifiers_ = new Set();
+
+    /** @private {number} */
+    this.rate_ = this.harness_.getRate();
+
+    /** @private {number} */
+    this.pollRate_ = 0.25;
+
+    /** @private {shaka.util.Timer} */
+    this.timer_ = new shaka.util.Timer(() => {
+      this.harness_.movePlayhead(this.rate_ * this.pollRate_);
+    });
+  }
+
+  /** @override */
+  release() {
+    if (this.timer_) {
+      this.timer_.stop();
+      this.timer_ = null;
+    }
+
+    this.harness_ = null;
+  }
+
+  /**
+   * Add a modifier to the controller. If the modifier has already been applied,
+   * this will be a no-op. Calling |removeModifier| will remove all instances of
+   * this modifier from the controller.
+   *
+   * @param {shaka.media.PlayRateController.Modifier} modifier
+   */
+  addModifier(modifier) {
+    this.modifiers_.add(modifier);
+    this.apply_();
+  }
+
+  /**
+   * Remove |modifier| from the controller. If the modifier was never applied,
+   * this will be a no-op. If the modifier was applied multiple times, this will
+   * remove all instances of it.
+   *
+   * @param {shaka.media.PlayRateController.Modifier} modifier
+   */
+  removeModifier(modifier) {
+    this.modifiers_.delete(modifier);
+    this.apply_();
+  }
+
+  /**
+   * Set the playback rate. This rate will only be used as provided when there
+   * are not modifiers. If the rate needs to be set to zero, it is suggested
+   * that you use the ZERO_RATE modifier instead.
+   *
+   * @param {number} rate
+   */
+  set(rate) {
+    this.rate_ = rate;
+    this.apply_();
+  }
+
+  /**
+   * Get the rate that the user will experience. This means that if we are using
+   * trick play, this will report the trick play rate. If we are buffering, this
+   * will report zero. If playback is occurring as normal, this will report 1.
+   *
+   * @return {number}
+   */
+  getActiveRate() {
+    return this.calculateCurrentRate_();
+  }
+
+  /**
+   * Reapply the effects of |this.rate_| and |this.active_| to the media
+   * element. This will only update the rate via the harness if the desired rate
+   * has changed.
+   *
+   * @private
+   */
+  apply_() {
+    // Always stop the timer. We may not start it again.
+    this.timer_.stop();
+
+    /** @type {number} */
+    const rate = this.calculateCurrentRate_();
+
+    if (rate >= 0) {
+      this.applyRate_(rate);
+      return;
+    }
+
+    // When moving backwards, set the playback rate to 0 so that we can manually
+    // seek backwards with out fighting the playhead.
+    this.timer_.tickEvery(this.pollRate_);
+    this.applyRate_(0);
+  }
+
+  /**
+   * Calculate the rate that the controller wants the media element to have
+   * based on the current state of the controller.
+   *
+   * @return {number}
+   * @private
+   */
+  calculateCurrentRate_() {
+    // We assume that all modifiers affect the play rate in the same way,
+    // setting it to zero. So if there are any modifiers, we should use a rate
+    // equal to zero.
+    return this.modifiers_.size ? 0 : this.rate_;
+  }
+
+  /**
+   * If the new rate is different than the media element's playback rate, this
+   * will change the playback rate. If the rate does not need to change, it will
+   * not be set. This will avoid unnecessary ratechange events.
+   *
+   * @param {number} newRate
+   * @return {boolean}
+   * @private
+   */
+  applyRate_(newRate) {
+    const oldRate = this.harness_.getRate();
+
+    if (oldRate != newRate) {
+      this.harness_.setRate(newRate);
+    }
+
+    return oldRate != newRate;
+  }
+};
+
+
+/**
+ * Modifiers are differ types of modifications that can be applied to the play
+ * rate controller than will affect what playback rate is used. Right now all
+ * modifiers will override the playback rate to 0.
+ *
+ * @enum {number}
+ */
+shaka.media.PlayRateController.Modifier = {
+  ZERO_RATE: 0,
+  BUFFERING: 1,
+};
+
+
+/**
+ * @typedef {{
+ *   getRate: function():number,
+ *   setRate: function(number),
+ *   movePlayhead: function(number)
+ * }}
+ *
+ * @description
+ *   A layer of abstraction between the controller and what it is controlling.
+ *   In tests this will be implemented with spies. In production this will be
+ *   implemented using a media element.
+ *
+ * @property {function():number} getRate
+ *   Get the current playback rate being seen by the user.
+ *
+ * @property {function(number)} setRate
+ *   Set the playback rate that the user should see.
+ *
+ * @property {function(number)} movePlayhead
+ *   Move the playhead N seconds. If N is positive, the playhead will move
+ *   forward abs(N) seconds. If N is negative, the playhead will move backwards
+ *   abs(N) seconds.
+ */
+shaka.media.PlayRateController.Harness;
diff --git a/lib/media/playhead.js b/lib/media/playhead.js
index 189e0abac0..03a06e6a4f 100644
--- a/lib/media/playhead.js
+++ b/lib/media/playhead.js
@@ -55,30 +55,6 @@ shaka.media.Playhead = class {
    */
   getTime() {}
 
-  /**
-   * Mark the playhead as buffering or not buffering.
-   *
-   * @param {boolean} isBuffering
-   */
-  setBuffering(isBuffering) {}
-
-  /**
-   * Get the playhead's playback rate.
-   *
-   * @return {number}
-   */
-  getPlaybackRate() {}
-
-  /**
-   * Set the playhead's playback rate. Setting to values greater than 1 will
-   * result in faster than normal playback. Setting to values less than 1 will
-   * return in slower than normal playback. Setting to 0 will stop playback.
-   * Setting to valids less than 0 is not allowed.
-   *
-   * @param {number} rate
-   */
-  setPlaybackRate(rate) {}
-
   /**
    * Notify the playhead that the buffered ranges have changed.
    */
@@ -103,10 +79,6 @@ shaka.media.SrcEqualsPlayhead = class {
     this.started_ = false;
     /** @private {?number} */
     this.startTime_ = null;
-    /** @private {number} */
-    this.playbackRate_ = 1;
-    /** @private {boolean} */
-    this.isBuffering_ = false;
 
     /** @private {shaka.util.EventManager} */
     this.eventManager_ = new shaka.util.EventManager();
@@ -157,50 +129,8 @@ shaka.media.SrcEqualsPlayhead = class {
     return time || 0;
   }
 
-  /** @override */
-  setBuffering(isBuffering) {
-    this.updateBufferingAndPlaybackRate_(isBuffering, null);
-  }
-
-  /** @override */
-  getPlaybackRate() {
-    // Always report our cached playback rate since it is what we should be
-    // playing at. If we are buffering, the media element's playback rate will
-    // be 0, but we are still planning to play at our cached playback rate.
-    return this.playbackRate_;
-  }
-
-  /** @override */
-  setPlaybackRate(rate) {
-    this.updateBufferingAndPlaybackRate_(null, rate);
-  }
-
   /** @override */
   notifyOfBufferingChange() {}
-
-  /**
-   * Update the internal state by setting the buffering and playback rate. When
-   * a value is set as |null|, the old value will be used. This will update the
-   * playback rate on the media element.
-   *
-   * @param {?boolean} buffering
-   * @param {?number} rate
-   */
-  updateBufferingAndPlaybackRate_(buffering, rate) {
-    if (buffering != null) {
-      this.isBuffering_ = buffering;
-    }
-
-    if (rate != null) {
-      this.playbackRate_ = rate;
-    }
-
-    // If we are buffering, we want playback rate to be 0, but once we stop
-    // buffering we want to reset the playback rate to what it was previously.
-    this.mediaElement_.playbackRate = this.isBuffering_ ?
-                                      0 :
-                                      this.playbackRate_;
-  }
 };
 
 
@@ -358,21 +288,6 @@ shaka.media.MediaSourcePlayhead = class {
     return this.clampSeekToDuration_(this.clampTime_(startTime));
   }
 
-  /** @override */
-  setBuffering(buffering) {
-    this.videoWrapper_.setBuffering(buffering);
-  }
-
-  /** @override */
-  getPlaybackRate() {
-    return this.videoWrapper_.getPlaybackRate();
-  }
-
-  /** @override */
-  setPlaybackRate(rate) {
-    this.videoWrapper_.setPlaybackRate(rate);
-  }
-
   /** @override */
   notifyOfBufferingChange() {
     this.gapController_.onSegmentAppended();
diff --git a/lib/media/video_wrapper.js b/lib/media/video_wrapper.js
index 9937606e54..93b757c41c 100644
--- a/lib/media/video_wrapper.js
+++ b/lib/media/video_wrapper.js
@@ -52,15 +52,6 @@ shaka.media.VideoWrapper = function(video, onSeek, startTime) {
   /** @private {shaka.util.EventManager} */
   this.eventManager_ = new shaka.util.EventManager();
 
-  /** @private {number} */
-  this.playbackRate_ = 1;
-
-  /** @private {boolean} */
-  this.buffering_ = false;
-
-  /** @private {shaka.util.Timer} */
-  this.trickPlayTimer_ = null;
-
   /** @private {shaka.media.VideoWrapper.PlayheadMover} */
   this.mover_ = new shaka.media.VideoWrapper.PlayheadMover(
       /* mediaElement= */ video,
@@ -75,8 +66,6 @@ shaka.media.VideoWrapper = function(video, onSeek, startTime) {
   } else {
     this.delaySetStartTime_(startTime);
   }
-
-  this.eventManager_.listen(video, 'ratechange', this.onRateChange_.bind(this));
 };
 
 
@@ -87,11 +76,6 @@ shaka.media.VideoWrapper.prototype.release = function() {
     this.eventManager_ = null;
   }
 
-  if (this.trickPlayTimer_ != null) {
-    this.trickPlayTimer_.stop();
-    this.trickPlayTimer_ = null;
-  }
-
   if (this.mover_ != null) {
     this.mover_.release();
     this.mover_ = null;
@@ -127,77 +111,6 @@ shaka.media.VideoWrapper.prototype.setTime = function(time) {
   }
 };
 
-
-/**
- * Gets the current effective playback rate.  This may be negative even if the
- * browser does not directly support rewinding.
- * @return {number}
- */
-shaka.media.VideoWrapper.prototype.getPlaybackRate = function() {
-  return this.playbackRate_;
-};
-
-
-/**
- * Sets the playback rate.
- * @param {number} rate
- */
-shaka.media.VideoWrapper.prototype.setPlaybackRate = function(rate) {
-  if (this.trickPlayTimer_ != null) {
-    this.trickPlayTimer_.stop();
-    this.trickPlayTimer_ = null;
-  }
-
-  this.playbackRate_ = rate;
-  // All major browsers support playback rates above zero.  Only need fake
-  // trick play for negative rates.
-  this.video_.playbackRate = (this.buffering_ || rate < 0) ? 0 : rate;
-
-  if (!this.buffering_ && rate < 0) {
-    // Defer creating the timer until we stop buffering.  This function will be
-    // called again from setBuffering().
-    this.trickPlayTimer_ = new shaka.util.Timer(() => {
-      this.video_.currentTime += rate / 4;
-    }).tickEvery(/* seconds= */ 0.25);
-  }
-};
-
-
-/**
- * Stops the playhead for buffering, or resumes the playhead after buffering.
- *
- * @param {boolean} buffering True to stop the playhead; false to allow it to
- *   continue.
- */
-shaka.media.VideoWrapper.prototype.setBuffering = function(buffering) {
-  if (buffering != this.buffering_) {
-    this.buffering_ = buffering;
-    this.setPlaybackRate(this.playbackRate_);
-  }
-};
-
-
-/**
- * Handles a 'ratechange' event.
- *
- * @private
- */
-shaka.media.VideoWrapper.prototype.onRateChange_ = function() {
-  // NOTE: This will not allow explicitly setting the playback rate to 0 while
-  // the playback rate is negative.  Pause will still work.
-  let expectedRate =
-      this.buffering_ || this.playbackRate_ < 0 ? 0 : this.playbackRate_;
-
-  // Native controls in Edge trigger a change to playbackRate and set it to 0
-  // when seeking.  If we don't exclude 0 from this check, we will force the
-  // rate to stay at 0 after a seek with Edge native controls.
-  // https://github.com/google/shaka-player/issues/951
-  if (this.video_.playbackRate && this.video_.playbackRate != expectedRate) {
-    shaka.log.debug('Video playback rate changed to', this.video_.playbackRate);
-    this.setPlaybackRate(this.video_.playbackRate);
-  }
-};
-
 /**
  * If the media element is not ready, we can't set |currentTime|. To work around
  * this we will listen for the "loadedmetadata" event so that we can set the
diff --git a/lib/player.js b/lib/player.js
index 9bb8b181a1..ac3e6de8c6 100644
--- a/lib/player.js
+++ b/lib/player.js
@@ -29,6 +29,7 @@ goog.require('shaka.media.MediaSourceEngine');
 goog.require('shaka.media.MuxJSClosedCaptionParser');
 goog.require('shaka.media.NoopCaptionParser');
 goog.require('shaka.media.PeriodObserver');
+goog.require('shaka.media.PlayRateController');
 goog.require('shaka.media.Playhead');
 goog.require('shaka.media.PlayheadObserverManager');
 goog.require('shaka.media.PreferenceBasedCriteria');
@@ -112,6 +113,15 @@ shaka.Player = function(mediaElement, dependencyInjector) {
    */
   this.playheadObservers_ = null;
 
+  /**
+   * This is our control over the playback rate of the media element. This
+   * provides the missing functionality that we need to provide trick play, for
+   * example a negative playback rate.
+   *
+   * @private {shaka.media.PlayRateController}
+   */
+  this.playRateController_ = null;
+
   // We use the buffering observer and timer to track when we move from having
   // enough buffered content to not enough. They only exist when content has
   // been loaded and are not re-used between loads.
@@ -1113,6 +1123,7 @@ shaka.Player.prototype.onUnload_ = async function(has, wants) {
     this.eventManager_.unlisten(has.mediaElement, 'playing');
     this.eventManager_.unlisten(has.mediaElement, 'pause');
     this.eventManager_.unlisten(has.mediaElement, 'ended');
+    this.eventManager_.unlisten(has.mediaElement, 'ratechange');
   }
 
   // Some observers use some playback components, shutting down the observers
@@ -1511,9 +1522,11 @@ shaka.Player.prototype.onLoad_ = async function(has, wants) {
   this.stats_ = new shaka.util.Stats();
 
   const updateStateHistory = () => this.updateStateHistory_();
+  const onRateChange = () => this.onRateChange_();
   this.eventManager_.listen(mediaElement, 'playing', updateStateHistory);
   this.eventManager_.listen(mediaElement, 'pause', updateStateHistory);
   this.eventManager_.listen(mediaElement, 'ended', updateStateHistory);
+  this.eventManager_.listen(mediaElement, 'ratechange', onRateChange);
 
   const AbrManagerFactory = this.config_.abrFactory;
   if (!this.abrManager_ || this.abrManagerFactory_ != AbrManagerFactory) {
@@ -1550,6 +1563,12 @@ shaka.Player.prototype.onLoad_ = async function(has, wants) {
   this.playhead_ = this.createPlayhead(has.startTime);
   this.playheadObservers_ = this.createPlayheadObserversForMSE_();
 
+  this.playRateController_ = new shaka.media.PlayRateController({
+    getRate: () => has.mediaElement.playbackRate,
+    setRate: (rate) => { has.mediaElement.playbackRate = rate; },
+    movePlayhead: (delta) => { has.mediaElement.currentTime += delta; },
+  });
+
   // We need to start the buffer management code near the end because it will
   // set the initial buffering state and that depends on other components being
   // initialized.
@@ -1671,6 +1690,12 @@ shaka.Player.prototype.onSrcEquals_ = function(has, wants) {
     this.playhead_.setStartTime(has.startTime);
   }
 
+  this.playRateController_ = new shaka.media.PlayRateController({
+    getRate: () => has.mediaElement.playbackRate,
+    setRate: (rate) => { has.mediaElement.playbackRate = rate; },
+    movePlayhead: (delta) => { has.mediaElement.currentTime += delta; },
+  });
+
   // We need to start the buffer management code near the end because it will
   // set the initial buffering state and that depends on other components being
   // initialized.
@@ -2433,9 +2458,9 @@ shaka.Player.prototype.isBuffering = function() {
 
 
 /**
- * Get the current effective playback rate. When trick play is enabled, this
- * will return the trick play rate. When trick play is disabled, this will
- * return the media element playback rate.
+ * Get the playback rate of what is playing right now. If we are using trick
+ * play, this will return the trick play rate. If no content is playing, this
+ * will return 0. If content is buffering, this will return 0.
  *
  * If the player has not loaded content or is still loading content, this will
  * return a playback rate of |0|.
@@ -2447,7 +2472,7 @@ shaka.Player.prototype.getPlaybackRate = function() {
   const loaded = this.loadMode_ == shaka.Player.LoadMode.MEDIA_SOURCE ||
                  this.loadMode_ == shaka.Player.LoadMode.SRC_EQUALS;
 
-  return loaded ? this.playhead_.getPlaybackRate() : 0;
+  return loaded ? this.playRateController_.getActiveRate() : 0;
 };
 
 
@@ -2466,13 +2491,14 @@ shaka.Player.prototype.getPlaybackRate = function() {
  * @export
  */
 shaka.Player.prototype.trickPlay = function(rate) {
-  if (this.loadMode_ != shaka.Player.LoadMode.MEDIA_SOURCE) {
-    return;
+  if (this.loadMode_ == shaka.Player.LoadMode.SRC_EQUALS) {
+    this.playRateController_.set(rate);
   }
 
-  shaka.log.debug('Trick play rate', rate);
-  this.playhead_.setPlaybackRate(rate);
-  this.streamingEngine_.setTrickPlay(rate != 1);
+  if (this.loadMode_ == shaka.Player.LoadMode.MEDIA_SOURCE) {
+    this.playRateController_.set(rate);
+    this.streamingEngine_.setTrickPlay(Math.abs(rate) > 1);
+  }
 };
 
 
@@ -2483,13 +2509,14 @@ shaka.Player.prototype.trickPlay = function(rate) {
  * @export
  */
 shaka.Player.prototype.cancelTrickPlay = function() {
-  if (this.loadMode_ != shaka.Player.LoadMode.MEDIA_SOURCE) {
-    return;
+  if (this.loadMode_ == shaka.Player.LoadMode.SRC_EQUALS) {
+    this.playRateController_.set(1);
   }
 
-  shaka.log.debug('Trick play canceled');
-  this.playhead_.setPlaybackRate(1);
-  this.streamingEngine_.setTrickPlay(false);
+  if (this.loadMode_ == shaka.Player.LoadMode.MEDIA_SOURCE) {
+    this.playRateController_.set(1);
+    this.streamingEngine_.setTrickPlay(false);
+  }
 };
 
 
@@ -3664,8 +3691,15 @@ shaka.Player.prototype.updateBufferState_ = function() {
 
   // TODO: Make the check for "loaded" simpler.
   if (loaded) {
+    const modifier = shaka.media.PlayRateController.Modifier.BUFFERING;
+
+    if (isBuffering) {
+      this.playRateController_.addModifier(modifier);
+    } else {
+      this.playRateController_.removeModifier(modifier);
+    }
+
     this.updateStateHistory_();
-    this.playhead_.setBuffering(isBuffering);
   }
 
   // Surface the buffering event so that the app knows if/when we are buffering.
@@ -3683,6 +3717,42 @@ shaka.Player.prototype.onChangePeriod_ = function() {
 };
 
 
+/**
+ * A callback for when the playback rate changes. We need to watch the playback
+ * rate so that if the playback rate on the media element changes (that was not
+ * caused by our play rate controller) we can notify the controller so that it
+ * can stay in-sync with the change.
+ *
+ * @private
+ */
+shaka.Player.prototype.onRateChange_ = function() {
+  const Modifier = shaka.media.PlayRateController.Modifier;
+
+  /** @type {number} */
+  const newRate = this.video_.playbackRate;
+
+  // On Edge, when someone seeks using the native controls, it will set the
+  // playback rate to zero until they finish seeking, after which it will
+  // return the playback rate.
+  //
+  // If the playback rate changes while seeking, Edge will cache the playback
+  // rate and use it after seeking.
+  //
+  // https://github.com/google/shaka-player/issues/951
+  if (newRate == 0) {
+    // Use a modifier so that we don't override our previous playback rate.
+    this.playRateController_.addModifier(Modifier.ZERO_RATE);
+    return;
+  }
+
+  // The playback rate has changed. This could be us or someone else. Make sure
+  // that there is no zero-rate modifier on the controller and set the new rate.
+  // If this was us, setting the rate again will be a no-op.
+  this.playRateController_.removeModifier(Modifier.ZERO_RATE);
+  this.playRateController_.set(newRate);
+};
+
+
 /**
  * Try updating the state history. If the player has not finished initializing,
  * this will be a no-op.
diff --git a/test/media/play_rate_controller_unit.js b/test/media/play_rate_controller_unit.js
new file mode 100644
index 0000000000..276c58d43d
--- /dev/null
+++ b/test/media/play_rate_controller_unit.js
@@ -0,0 +1,151 @@
+/**
+ * @license
+ * Copyright 2016 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+describe('PlayRateController', () => {
+  const Modifier = shaka.media.PlayRateController.Modifier;
+
+  /** @type {!jasmine.Spy} */
+  let getPlayRateSpy;
+  /** @type {!jasmine.Spy} */
+  let setPlayRateSpy;
+  /** @type {!jasmine.Spy} */
+  let movePlayheadSpy;
+
+  /** @type {number} */
+  let playRate;
+
+  /** @type {!shaka.media.PlayRateController} */
+  let controller;
+
+  beforeEach(() => {
+    getPlayRateSpy = jasmine.createSpy('getPlaybackRate');
+    setPlayRateSpy = jasmine.createSpy('setPlaybackRate');
+    movePlayheadSpy = jasmine.createSpy('movePlayhead');
+
+    playRate = 1;
+
+    getPlayRateSpy.and.callFake(() => playRate);
+    setPlayRateSpy.and.callFake((rate) => { playRate = rate; });
+
+    const harness = {
+      getRate: shaka.test.Util.spyFunc(getPlayRateSpy),
+      setRate: shaka.test.Util.spyFunc(setPlayRateSpy),
+      movePlayhead: shaka.test.Util.spyFunc(movePlayheadSpy),
+    };
+
+    controller = new shaka.media.PlayRateController(harness);
+  });
+
+  // When the playback rate is positive, we want to see that the media element's
+  // playback rate gets set to the playback rate.
+  it('positive playback rate', () => {
+    controller.set(5);
+    expect(setPlayRateSpy).toHaveBeenCalledWith(5);
+  });
+
+  // When the playback rate is negative, we want to see the media element's
+  // playback rate get set to zero.
+  it('negative playback rate', () => {
+    controller.set(-5);
+    expect(setPlayRateSpy).toHaveBeenCalledWith(0);
+  });
+
+  it('buffering modifier sets rate to zero', () => {
+    controller.addModifier(Modifier.BUFFERING);
+    expect(setPlayRateSpy).toHaveBeenCalledWith(0);
+
+    setPlayRateSpy.calls.reset();
+
+    controller.removeModifier(Modifier.BUFFERING);
+    expect(setPlayRateSpy).toHaveBeenCalledWith(1);
+  });
+
+  it('zero rate modifier sets rate to zero', () => {
+    controller.addModifier(Modifier.ZERO_RATE);
+    expect(setPlayRateSpy).toHaveBeenCalledWith(0);
+
+    setPlayRateSpy.calls.reset();
+
+    controller.removeModifier(Modifier.ZERO_RATE);
+    expect(setPlayRateSpy).toHaveBeenCalledWith(1);
+  });
+
+  it('adding duplicate modifier has no effect', () => {
+    controller.addModifier(Modifier.ZERO_RATE);
+    expect(setPlayRateSpy).toHaveBeenCalledWith(0);
+
+    // Reset the calls so that we can make sure it was not called again.
+    setPlayRateSpy.calls.reset();
+
+    controller.addModifier(Modifier.ZERO_RATE);
+    expect(setPlayRateSpy).not.toHaveBeenCalled();
+  });
+
+  it('removing absent modifier has no effect', () => {
+    controller.removeModifier(Modifier.ZERO_RATE);
+    expect(setPlayRateSpy).not.toHaveBeenCalled();
+  });
+
+  it('modifiers work with eacher other', () => {
+    controller.addModifier(Modifier.BUFFERING);
+    controller.addModifier(Modifier.ZERO_RATE);
+
+    expect(setPlayRateSpy).toHaveBeenCalledWith(0);
+
+    // Reset the calls so that we can check that no other calls were made.
+    setPlayRateSpy.calls.reset();
+
+    controller.removeModifier(Modifier.BUFFERING);
+
+    // The rates should not have changed since there is still a modifier in
+    // place.
+    expect(setPlayRateSpy).not.toHaveBeenCalled();
+
+    controller.removeModifier(Modifier.ZERO_RATE);
+
+    expect(setPlayRateSpy).toHaveBeenCalledWith(1);
+  });
+
+  // When we set the rate while modifiers are in-place, we should see the new
+  // rate be used once the modifiers are removed.
+  it('set takes effect after modifiers are removed', () => {
+    controller.addModifier(Modifier.BUFFERING);
+
+    expect(setPlayRateSpy).toHaveBeenCalledWith(0);
+
+    // Reset so that we can make sure it was not called after we call |set(4)|.
+    setPlayRateSpy.calls.reset();
+
+    controller.set(4);
+
+    expect(setPlayRateSpy).not.toHaveBeenCalled();
+    controller.removeModifier(Modifier.BUFFERING);
+
+    expect(setPlayRateSpy).toHaveBeenCalledWith(4);
+  });
+
+  // Make sure that when the playback rate set, if the new rate matches the
+  // current rate, the controller will not set the rate on the media element.
+  it('does not redundently set the playrate', ()=> {
+    // Make sure we don't see the play rate change before and after we set the
+    // rate on the controller.
+    expect(setPlayRateSpy).not.toHaveBeenCalled();
+    controller.set(1);
+    expect(setPlayRateSpy).not.toHaveBeenCalled();
+  });
+});
diff --git a/test/media/playhead_unit.js b/test/media/playhead_unit.js
index 3f98e559a3..c42c25c048 100644
--- a/test/media/playhead_unit.js
+++ b/test/media/playhead_unit.js
@@ -1301,69 +1301,4 @@ describe('Playhead', function() {
       return HTMLMediaElement.HAVE_METADATA;
     }
   });  // gap jumping
-
-  describe('rate changes', function() {
-    beforeEach(function() {
-      playhead = new shaka.media.MediaSourcePlayhead(
-          video,
-          manifest,
-          config,
-          0 /* startTime */,
-          Util.spyFunc(onSeek),
-          Util.spyFunc(onEvent));
-    });
-
-    it('notices video rate changes', function() {
-      expect(playhead.getPlaybackRate()).toBe(1);
-
-      video.playbackRate = 2;
-      video.on['ratechange']();
-      expect(playhead.getPlaybackRate()).toBe(2);
-    });
-
-    it('controls video rate with setPlaybackRate', function() {
-      expect(playhead.getPlaybackRate()).toBe(1);
-      playhead.setPlaybackRate(2);
-      expect(playhead.getPlaybackRate()).toBe(2);
-      expect(video.playbackRate).toBe(2);
-    });
-
-    it('sets video rate to 0 when buffering', function() {
-      expect(video.playbackRate).toBe(1);
-      playhead.setBuffering(true);
-      expect(video.playbackRate).toBe(0);
-    });
-
-    it('remembers previous rate while buffering', function() {
-      playhead.setPlaybackRate(5);
-      expect(video.playbackRate).toBe(5);
-      expect(playhead.getPlaybackRate()).toBe(5);
-      playhead.setBuffering(true);
-      expect(video.playbackRate).toBe(0);
-      expect(playhead.getPlaybackRate()).toBe(5);
-      playhead.setBuffering(false);
-      expect(video.playbackRate).toBe(5);
-      expect(playhead.getPlaybackRate()).toBe(5);
-    });
-
-    it('ignores a rate change to 0', function() {
-      // Regression test for https://github.com/google/shaka-player/issues/951
-      expect(video.playbackRate).toBe(1);
-      expect(playhead.getPlaybackRate()).toBe(1);
-
-      // With native controls on Edge, a rate change to 0 occurs when the user
-      // seeks.  This seems to happen before setBuffering(true).
-      video.playbackRate = 0;
-      video.on['ratechange']();
-      expect(playhead.getPlaybackRate()).toBe(1);
-
-      playhead.setBuffering(true);
-      expect(video.playbackRate).toBe(0);
-      expect(playhead.getPlaybackRate()).toBe(1);
-
-      playhead.setBuffering(false);
-      expect(video.playbackRate).toBe(1);
-      expect(playhead.getPlaybackRate()).toBe(1);
-    });
-  });  // rate changes
 });