diff --git a/.wp-env.json b/.wp-env.json
index 90457c8da459e..79810b194b667 100644
--- a/.wp-env.json
+++ b/.wp-env.json
@@ -10,8 +10,7 @@
 				"wp-content/plugins/gutenberg-test-plugins": "./packages/e2e-tests/plugins",
 				"wp-content/themes/gutenberg-test-themes": "./test/gutenberg-test-themes",
 				"wp-content/themes/gutenberg-test-themes/twentytwentyone": "https://downloads.wordpress.org/theme/twentytwentyone.1.7.zip",
-				"wp-content/themes/gutenberg-test-themes/twentytwentythree": "https://downloads.wordpress.org/theme/twentytwentythree.1.0.zip",
-				"wp-content/themes/gutenberg-test-themes/twentytwentyfour": "https://downloads.wordpress.org/theme/twentytwentyfour.1.0.zip"
+				"wp-content/themes/gutenberg-test-themes/twentytwentythree": "https://downloads.wordpress.org/theme/twentytwentythree.1.0.zip"
 			}
 		}
 	}
diff --git a/test/performance/specs/site-editor.spec.js b/test/performance/specs/site-editor.spec.js
index 458d41cbeb9a0..38bcceb14edd6 100644
--- a/test/performance/specs/site-editor.spec.js
+++ b/test/performance/specs/site-editor.spec.js
@@ -28,7 +28,6 @@ const results = {
 	inserterSearch: [],
 	listViewOpen: [],
 	navigate: [],
-	loadPatterns: [],
 };
 
 test.describe( 'Site Editor Performance', () => {
@@ -207,83 +206,6 @@ test.describe( 'Site Editor Performance', () => {
 			} );
 		}
 	} );
-
-	test.describe( 'Loading Patterns', () => {
-		test.beforeAll( async ( { requestUtils } ) => {
-			await requestUtils.activateTheme( 'twentytwentyfour' );
-		} );
-
-		test.afterAll( async ( { requestUtils } ) => {
-			await requestUtils.activateTheme( 'twentytwentyfour' );
-		} );
-
-		test( 'Run the test', async ( { page, admin, perfUtils, editor } ) => {
-			const samples = 10;
-			for ( let i = 1; i <= samples; i++ ) {
-				// We want to start from a fresh state each time, without
-				// queries or patterns already cached.
-				await admin.visitSiteEditor( {
-					postId: 'twentytwentyfour//home',
-					postType: 'wp_template',
-					canvas: 'edit',
-				} );
-				await editor.openDocumentSettingsSidebar();
-				await page
-					.getByRole( 'button', {
-						name: 'Actions',
-					} )
-					.click();
-
-				// Wait for the browser to be idle before starting the monitoring.
-				// eslint-disable-next-line no-restricted-syntax
-				await page.waitForTimeout( BROWSER_IDLE_WAIT );
-
-				const startTime = performance.now();
-
-				await page
-					.getByRole( 'menuitem', { name: 'Replace template' } )
-					.click();
-
-				const patterns = [
-					'Blogging home template',
-					'Business home template',
-					'Portfolio home template with post featured images',
-					'Blogging index template',
-				];
-
-				await Promise.all(
-					patterns.map( async ( pattern ) => {
-						const canvas = await perfUtils.getCanvas(
-							page
-								.getByRole( 'option', {
-									name: pattern,
-									exact: true,
-								} )
-								.getByTitle( 'Editor canvas' )
-						);
-
-						// Wait until the first block is rendered AND all
-						// patterns are replaced.
-						await Promise.all( [
-							canvas.locator( '.wp-block' ).first().waitFor(),
-							page.waitForFunction(
-								() =>
-									document.querySelectorAll(
-										'[data-type="core/pattern"]'
-									).length === 0
-							),
-						] );
-					} )
-				);
-
-				const endTime = performance.now();
-
-				results.loadPatterns.push( endTime - startTime );
-
-				await page.keyboard.press( 'Escape' );
-			}
-		} );
-	} );
 } );
 
 /* eslint-enable playwright/no-conditional-in-test, playwright/expect-expect */