diff --git a/index.js b/index.js
index 6322c89..f88dd07 100644
--- a/index.js
+++ b/index.js
@@ -44,6 +44,7 @@ export default function pTimeout(promise, options) {
 	} = options;
 
 	let timer;
+	let abortHandler;
 
 	const wrappedPromise = new Promise((resolve, reject) => {
 		if (typeof milliseconds !== 'number' || Math.sign(milliseconds) !== 1) {
@@ -56,15 +57,11 @@ export default function pTimeout(promise, options) {
 				reject(getAbortedReason(signal));
 			}
 
-			const abortHandler = () => {
+			abortHandler = () => {
 				reject(getAbortedReason(signal));
 			};
 
 			signal.addEventListener('abort', abortHandler, {once: true});
-
-			promise.finally(() => {
-				signal.removeEventListener('abort', abortHandler);
-			});
 		}
 
 		if (milliseconds === Number.POSITIVE_INFINITY) {
@@ -111,6 +108,9 @@ export default function pTimeout(promise, options) {
 
 	const cancelablePromise = wrappedPromise.finally(() => {
 		cancelablePromise.clear();
+		if (abortHandler && options.signal) {
+			options.signal.removeEventListener('abort', abortHandler);
+		}
 	});
 
 	cancelablePromise.clear = () => {
diff --git a/test.js b/test.js
index e2f5486..d9b6eb9 100644
--- a/test.js
+++ b/test.js
@@ -165,4 +165,31 @@ if (globalThis.AbortController !== undefined) {
 		addEventListenerSpy.restore();
 		removeEventListenerSpy.restore();
 	});
+
+	test('removes abort listener after promise rejects', async t => {
+		const abortController = new AbortController();
+		const {signal} = abortController;
+
+		const addEventListenerSpy = sinon.spy(signal, 'addEventListener');
+		const removeEventListenerSpy = sinon.spy(signal, 'removeEventListener');
+
+		const promise = pTimeout(
+			(async () => {
+				await delay(50);
+				throw new Error('Test error');
+			})(),
+			{
+				milliseconds: 100,
+				signal,
+			},
+		);
+
+		await t.throwsAsync(promise, {message: 'Test error'});
+
+		t.true(addEventListenerSpy.calledWith('abort'), 'addEventListener should be called with "abort"');
+		t.true(removeEventListenerSpy.calledWith('abort'), 'removeEventListener should be called with "abort"');
+
+		addEventListenerSpy.restore();
+		removeEventListenerSpy.restore();
+	});
 }