diff --git a/changelog.d/5342.bugfix b/changelog.d/5342.bugfix
new file mode 100644
index 000000000000..66a3076292f6
--- /dev/null
+++ b/changelog.d/5342.bugfix
@@ -0,0 +1 @@
+Fix failure when fetching batches of events during backfill, etc.
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index f3fc897a0ac6..7a4fa36e7cbd 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -280,6 +280,7 @@ def get_pdu(self, destinations, event_id, room_version, outlier=False,
                     "Failed to get PDU %s from %s because %s",
                     event_id, destination, e,
                 )
+                continue
             except NotRetryingDestination as e:
                 logger.info(str(e))
                 continue