diff --git a/packages/block-editor/src/components/block-patterns-list/index.js b/packages/block-editor/src/components/block-patterns-list/index.js
index 2609cc2db97a1..0ca27c839ad71 100644
--- a/packages/block-editor/src/components/block-patterns-list/index.js
+++ b/packages/block-editor/src/components/block-patterns-list/index.js
@@ -23,7 +23,6 @@ import { Icon, symbol } from '@wordpress/icons';
  */
 import BlockPreview from '../block-preview';
 import InserterDraggableBlocks from '../inserter-draggable-blocks';
-import BlockPatternsPaging from '../block-patterns-paging';
 import { INSERTER_PATTERN_TYPES } from '../inserter/block-patterns-tab/utils';
 
 const WithToolTip = ( { showTooltip, title, children } ) => {
@@ -177,17 +176,10 @@ function BlockPattern( {
 	);
 }
 
-function BlockPatternPlaceholder() {
-	return (
-		<div className="block-editor-block-patterns-list__item is-placeholder" />
-	);
-}
-
 function BlockPatternsList(
 	{
 		isDraggable,
 		blockPatterns,
-		shownPatterns,
 		onHover,
 		onClickPattern,
 		orientation,
@@ -195,7 +187,6 @@ function BlockPatternsList(
 		category,
 		showTitle = true,
 		showTitlesAsTooltip,
-		pagingProps,
 	},
 	ref
 ) {
@@ -205,11 +196,9 @@ function BlockPatternsList(
 		// Reset the active composite item whenever the available patterns change,
 		// to make sure that Composite widget can receive focus correctly when its
 		// composite items change. The first composite item will receive focus.
-		const firstCompositeItemId = blockPatterns.find( ( pattern ) =>
-			shownPatterns.includes( pattern )
-		)?.name;
+		const firstCompositeItemId = blockPatterns[ 0 ]?.name;
 		setActiveCompositeId( firstCompositeItemId );
-	}, [ shownPatterns, blockPatterns ] );
+	}, [ blockPatterns ] );
 
 	return (
 		<Composite
@@ -222,8 +211,7 @@ function BlockPatternsList(
 			ref={ ref }
 		>
 			{ blockPatterns.map( ( pattern ) => {
-				const isShown = shownPatterns.includes( pattern );
-				return isShown ? (
+				return (
 					<BlockPattern
 						key={ pattern.name }
 						id={ pattern.name }
@@ -235,11 +223,8 @@ function BlockPatternsList(
 						showTooltip={ showTitlesAsTooltip }
 						category={ category }
 					/>
-				) : (
-					<BlockPatternPlaceholder key={ pattern.name } />
 				);
 			} ) }
-			{ pagingProps && <BlockPatternsPaging { ...pagingProps } /> }
 		</Composite>
 	);
 }
diff --git a/packages/block-editor/src/components/inserter/block-patterns-explorer/pattern-list.js b/packages/block-editor/src/components/inserter/block-patterns-explorer/pattern-list.js
index 709e005b587ce..296b432bd685c 100644
--- a/packages/block-editor/src/components/inserter/block-patterns-explorer/pattern-list.js
+++ b/packages/block-editor/src/components/inserter/block-patterns-explorer/pattern-list.js
@@ -151,9 +151,6 @@ function PatternList( {
 				{ hasItems && (
 					<>
 						<BlockPatternsList
-							shownPatterns={
-								pagingProps.categoryPatternsAsyncList
-							}
 							blockPatterns={ pagingProps.categoryPatterns }
 							onClickPattern={ onClickPattern }
 							isDraggable={ false }
diff --git a/packages/block-editor/src/components/inserter/block-patterns-tab/pattern-category-previews.js b/packages/block-editor/src/components/inserter/block-patterns-tab/pattern-category-previews.js
index 61716f616dafa..bdfc7d7b64ca0 100644
--- a/packages/block-editor/src/components/inserter/block-patterns-tab/pattern-category-previews.js
+++ b/packages/block-editor/src/components/inserter/block-patterns-tab/pattern-category-previews.js
@@ -24,6 +24,7 @@ import { useSelect } from '@wordpress/data';
  */
 import usePatternsState from '../hooks/use-patterns-state';
 import BlockPatternsList from '../../block-patterns-list';
+import BlockPatternsPaging from '../../block-patterns-paging';
 import usePatternsPaging from '../hooks/use-patterns-paging';
 import { PatternsFilter } from './patterns-filter';
 import { usePatternCategories } from './use-pattern-categories';
@@ -183,7 +184,6 @@ export function PatternCategoryPreviews( {
 					) }
 					<BlockPatternsList
 						ref={ scrollContainerRef }
-						shownPatterns={ pagingProps.categoryPatternsAsyncList }
 						blockPatterns={ pagingProps.categoryPatterns }
 						onClickPattern={ onClickPattern }
 						onHover={ onHover }
@@ -193,8 +193,15 @@ export function PatternCategoryPreviews( {
 						isDraggable
 						showTitlesAsTooltip={ showTitlesAsTooltip }
 						patternFilter={ patternSourceFilter }
-						pagingProps={ pagingProps }
 					/>
+					{ pagingProps && (
+						<VStack
+							spacing={ 2 }
+							className="block-editor-inserter__patterns-category-panel-footer"
+						>
+							<BlockPatternsPaging { ...pagingProps } />
+						</VStack>
+					) }
 				</>
 			) }
 		</>
diff --git a/packages/block-editor/src/components/inserter/hooks/use-patterns-paging.js b/packages/block-editor/src/components/inserter/hooks/use-patterns-paging.js
index 931796acceeb0..cdffef4a00513 100644
--- a/packages/block-editor/src/components/inserter/hooks/use-patterns-paging.js
+++ b/packages/block-editor/src/components/inserter/hooks/use-patterns-paging.js
@@ -2,11 +2,10 @@
  * WordPress dependencies
  */
 import { useMemo, useState, useEffect } from '@wordpress/element';
-import { useAsyncList, usePrevious } from '@wordpress/compose';
+import { usePrevious } from '@wordpress/compose';
 import { getScrollContainer } from '@wordpress/dom';
 
 const PAGE_SIZE = 20;
-const INITIAL_INSERTER_RESULTS = 5;
 
 /**
  * Supplies values needed to page the patterns list client side.
@@ -42,9 +41,6 @@ export default function usePatternsPaging(
 			pageIndex * PAGE_SIZE + PAGE_SIZE
 		);
 	}, [ pageIndex, currentCategoryPatterns ] );
-	const categoryPatternsAsyncList = useAsyncList( categoryPatterns, {
-		step: INITIAL_INSERTER_RESULTS,
-	} );
 	const numPages = Math.ceil( currentCategoryPatterns.length / PAGE_SIZE );
 	const changePage = ( page ) => {
 		const scrollContainer = getScrollContainer(
@@ -68,7 +64,6 @@ export default function usePatternsPaging(
 	return {
 		totalItems,
 		categoryPatterns,
-		categoryPatternsAsyncList,
 		numPages,
 		changePage,
 		currentPage,
diff --git a/packages/block-editor/src/components/inserter/style.scss b/packages/block-editor/src/components/inserter/style.scss
index 9e727b1379524..fd9f54bad6633 100644
--- a/packages/block-editor/src/components/inserter/style.scss
+++ b/packages/block-editor/src/components/inserter/style.scss
@@ -249,7 +249,8 @@ $block-inserter-tabs-height: 44px;
 	}
 }
 
-.block-editor-inserter__patterns-category-panel-header {
+.block-editor-inserter__patterns-category-panel-header,
+.block-editor-inserter__patterns-category-panel-footer {
 	padding: $grid-unit-10 0;
 	@include break-medium {
 		padding: $grid-unit-10 $grid-unit-30;