diff --git a/packages/format-library/src/link/index.js b/packages/format-library/src/link/index.js
index 61b6f974cdce3a..bde0b14c5d962c 100644
--- a/packages/format-library/src/link/index.js
+++ b/packages/format-library/src/link/index.js
@@ -48,7 +48,7 @@ function Edit( {
 			return;
 		}
 
-		function handleClick( event ) {
+		function handleMouseUp( event ) {
 			// There is a situation whereby there is an existing link in the rich text
 			// and the user clicks on the leftmost edge of that link and fails to activate
 			// the link format, but the click event still fires on the `<a>` element.
@@ -62,10 +62,22 @@ function Edit( {
 			setAddingLink( true );
 		}
 
-		editableContentElement.addEventListener( 'click', handleClick );
+		const handleMouseDown = () => {
+			setAddingLink( false );
+		};
+
+		editableContentElement.addEventListener( 'mousedown', handleMouseDown );
+		editableContentElement.addEventListener( 'mouseup', handleMouseUp );
 
 		return () => {
-			editableContentElement.removeEventListener( 'click', handleClick );
+			editableContentElement.removeEventListener(
+				'mousedown',
+				handleMouseDown
+			);
+			editableContentElement.removeEventListener(
+				'mouseup',
+				handleMouseUp
+			);
 		};
 	}, [ contentRef, isActive ] );