diff --git a/glue_jupyter/bqplot/image/frb_mark.py b/glue_jupyter/bqplot/image/frb_mark.py
index 6327d614..564d75f5 100644
--- a/glue_jupyter/bqplot/image/frb_mark.py
+++ b/glue_jupyter/bqplot/image/frb_mark.py
@@ -38,6 +38,8 @@ def __init__(self, viewer, array_maker, compression='png'):
         self.viewer.figure.axes[1].scale.observe(self.debounced_update, 'min')
         self.viewer.figure.axes[1].scale.observe(self.debounced_update, 'max')
 
+        self._latest_hash = None
+
         # NOTE: we deliberately don't call .update() here because when FRBImage
         # is created for the main composite image layer the composite arrays
         # haven't been set up yet, and for subset layers the layer gets force
@@ -62,7 +64,7 @@ def external_padding(self, value):
         if value > previous_value:  # no point updating if the value is smaller than before
             self.debounced_update()
 
-    def update(self, *args, **kwargs):
+    def update(self, *args, force=False, **kwargs):
 
         # Shape can be (0, 0) when viewer was created and then destroyed.
         if self.shape is None or np.allclose(self.shape, 0):
@@ -77,6 +79,11 @@ def update(self, *args, **kwargs):
         if xmin is None or xmax is None or ymin is None or ymax is None:
             return
 
+        current_hash = (xmin, xmax, ymin, ymax, self.external_padding)
+
+        if not force and current_hash == self._latest_hash:
+            return
+
         ny, nx = self.shape
 
         # Expand beyond the boundary
@@ -103,5 +110,7 @@ def update(self, *args, **kwargs):
         else:
             self.image = EMPTY_IMAGE
 
+        self._latest_hash = current_hash
+
     def invalidate_cache(self):
-        self.update()
+        self.update(force=True)
diff --git a/glue_jupyter/bqplot/image/viewer.py b/glue_jupyter/bqplot/image/viewer.py
index 16895b1e..40444015 100644
--- a/glue_jupyter/bqplot/image/viewer.py
+++ b/glue_jupyter/bqplot/image/viewer.py
@@ -54,6 +54,15 @@ def __init__(self, session, compression='png'):
         on_change([(self.state, 'aspect')])(self._sync_figure_aspect)
         self._sync_figure_aspect()
 
+    def _update_bqplot_limits(self, *args, **kwargs):
+        # When the user explicitly changes the limits, we want the update to
+        # be immediate - debouncing should be ideally used mostly for preventing
+        # many successive updates from the front-end, e.g. when panning, but
+        # programmatically changing things should be immediate.
+        super()._update_bqplot_limits(*args, **kwargs)
+        if hasattr(self, '_composite_image'):
+            self._composite_image.update()
+
     def _update_axes(self, *args):
 
         if self.state.x_att_world is not None: