diff --git a/packages/block-editor/src/components/block-list/block.js b/packages/block-editor/src/components/block-list/block.js
index bdf33c944deb13..bb5658a558d98b 100644
--- a/packages/block-editor/src/components/block-list/block.js
+++ b/packages/block-editor/src/components/block-list/block.js
@@ -574,7 +574,6 @@ function BlockListBlockProvider( props ) {
 				hasBlockMovingClientId,
 				canInsertBlockType,
 				__unstableHasActiveBlockOverlayActive,
-				__unstableGetEditorMode,
 				getSelectedBlocksInitialCaretPosition,
 			} = unlock( select( blockEditorStore ) );
 			const blockWithoutAttributes =
@@ -678,12 +677,9 @@ function BlockListBlockProvider( props ) {
 				hasOverlay:
 					__unstableHasActiveBlockOverlayActive( clientId ) &&
 					! isDragging(),
-				initialPosition:
-					_isSelected &&
-					( __unstableGetEditorMode() === 'edit' ||
-						__unstableGetEditorMode() === 'zoom-out' ) // Don't recalculate the initialPosition when toggling in/out of zoom-out mode
-						? getSelectedBlocksInitialCaretPosition()
-						: undefined,
+				initialPosition: _isSelected
+					? getSelectedBlocksInitialCaretPosition()
+					: undefined,
 				isHighlighted: isBlockHighlighted( clientId ),
 				isMultiSelected,
 				isPartiallySelected:
diff --git a/packages/block-editor/src/components/block-list/use-block-props/use-nav-mode-exit.js b/packages/block-editor/src/components/block-list/use-block-props/use-nav-mode-exit.js
index aa9c0a630c5bd7..a7e9f26842704e 100644
--- a/packages/block-editor/src/components/block-list/use-block-props/use-nav-mode-exit.js
+++ b/packages/block-editor/src/components/block-list/use-block-props/use-nav-mode-exit.js
@@ -22,14 +22,12 @@ export function useNavModeExit( clientId ) {
 			function onMouseDown( event ) {
 				// Don't select a block if it's already handled by a child
 				// block.
-				if ( isNavigationMode() && ! event.defaultPrevented ) {
-					// Prevent focus from moving to the block.
-					event.preventDefault();
+				event.stopPropagation();
 
-					// When clicking on a selected block, exit navigation mode.
-					if ( isBlockSelected( clientId ) ) {
-						setNavigationMode( false );
-					} else {
+				if ( isNavigationMode() && ! event.defaultPrevented ) {
+					if ( ! isBlockSelected( clientId ) ) {
+						// Prevent focus from moving to the block.
+						event.preventDefault();
 						selectBlock( clientId );
 					}
 				}
diff --git a/packages/block-editor/src/components/block-tools/block-selection-button.js b/packages/block-editor/src/components/block-tools/block-selection-button.js
index 9c6c22181ef2ac..4ed80845aa8e7a 100644
--- a/packages/block-editor/src/components/block-tools/block-selection-button.js
+++ b/packages/block-editor/src/components/block-tools/block-selection-button.js
@@ -9,7 +9,7 @@ import clsx from 'clsx';
 import { dragHandle } from '@wordpress/icons';
 import { Button, Flex, FlexItem } from '@wordpress/components';
 import { useSelect, useDispatch } from '@wordpress/data';
-import { forwardRef, useEffect } from '@wordpress/element';
+import { forwardRef, useEffect, useRef } from '@wordpress/element';
 import {
 	BACKSPACE,
 	DELETE,
@@ -90,15 +90,20 @@ function BlockSelectionButton( { clientId, rootClientId }, ref ) {
 		[ clientId, rootClientId ]
 	);
 	const { label, icon, blockMovingMode, editorMode, canMove } = selected;
-	const { setNavigationMode, removeBlock } = useDispatch( blockEditorStore );
+	const { removeBlock } = useDispatch( blockEditorStore );
 
 	// Focus the breadcrumb in navigation mode.
+	const labelRef = useRef( label );
+	useEffect( () => {
+		labelRef.current = label;
+	}, [ label ] );
 	useEffect( () => {
 		if ( editorMode === 'navigation' ) {
 			ref.current.focus();
-			speak( label );
+			speak( labelRef.current );
 		}
-	}, [ label, editorMode ] );
+	}, [ editorMode ] );
+
 	const blockElement = useBlockElement( clientId );
 
 	const {
@@ -279,7 +284,7 @@ function BlockSelectionButton( { clientId, rootClientId }, ref ) {
 							ref={ ref }
 							onClick={
 								editorMode === 'navigation'
-									? () => setNavigationMode( false )
+									? () => blockElement?.focus()
 									: undefined
 							}
 							onKeyDown={ onKeyDown }
diff --git a/packages/block-editor/src/store/reducer.js b/packages/block-editor/src/store/reducer.js
index 91d853a50a6a42..90c69a91bda5b9 100644
--- a/packages/block-editor/src/store/reducer.js
+++ b/packages/block-editor/src/store/reducer.js
@@ -1797,11 +1797,6 @@ export const blockListSettings = ( state = {}, action ) => {
  * @return {string} Updated state.
  */
 export function editorMode( state = 'edit', action ) {
-	// Let inserting block in navigation mode always trigger Edit mode.
-	if ( action.type === 'INSERT_BLOCKS' && state === 'navigation' ) {
-		return 'edit';
-	}
-
 	if ( action.type === 'SET_EDITOR_MODE' ) {
 		return action.mode;
 	}