diff --git a/packages/block-editor/src/components/block-list/use-multi-selection.js b/packages/block-editor/src/components/block-list/use-multi-selection.js
index 1e77f8711a51de..8505ce7a075562 100644
--- a/packages/block-editor/src/components/block-list/use-multi-selection.js
+++ b/packages/block-editor/src/components/block-list/use-multi-selection.js
@@ -242,6 +242,17 @@ export default function useMultiSelection( ref ) {
 
 			startClientId.current = clientId;
 			anchorElement.current = document.activeElement;
+			if ( anchorElement.current ) {
+				const blockInspector = document.querySelector(
+					'.block-editor-block-inspector'
+				);
+				if (
+					blockInspector &&
+					blockInspector.contains( anchorElement.current )
+				) {
+					return;
+				}
+			}
 			startMultiSelect();
 
 			// `onSelectionStart` is called after `mousedown` and `mouseleave`