Refactor Page Paths utils and Middleware Plugin (#36576)

This PR brings some significant refactoring in preparation for upcoming middleware changes. Each commit can be reviewed independently, here is a summary of what each one does and the reasoning behind it:
- [Move pagesDir to next-dev-server](javivelasco@f2fe154) simply moves the `pagesDir` property to the dev server which is the only place where it is needed. Having it for every server is misleading.
- [Move (de)normalize page path utils to a file page-path-utils.ts](javivelasco@27cedf0) Moves the functions to normalize and denormalize page paths to a single file that is intended to hold every utility function that transforms page paths. Since those are complementary it makes sense to have them together. I also added explanatory comments on why they are not idempotent and examples for input -> output that I find very useful.
- [Extract removePagePathTail](javivelasco@6b12133) This extracts a function to remove the tail on a page path (absolute or relative). I'm sure there will be other contexts where we can use it.
- [Extract getPagePaths and refactor findPageFile](javivelasco@cf2c7b8) This extracts a function `getPagePaths` that is used to generate an array of paths to inspect when looking for a page file from `findPageFile`. Then it refactors such function to use it parallelizing lookups. This will allow us to print every path we look at when looking for a file which can be useful for debugging. It also adds a `flatten` helper. 
- [Refactor onDemandEntryHandler](javivelasco@4be685c) I've found this one quite difficult to understand so it is refactored to use some of the previously mentioned functions and make it easier to read.
- [Extract absolutePagePath util](javivelasco@3bc0783) Extracts yet another util from the `next-dev-server` that transforms an absolute path into a page name. Of course it adds comments, parameters and examples.
- [Refactor MiddlewarePlugin](javivelasco@c595a2c) This is the most significant change. The logic here was very hard to understand so it is totally redistributed with comments. This also removes a global variable `ssrEntries` that was deprecated in favour of module metadata added to Webpack from loaders keeping less dependencies. It also adds types and makes a clear distinction between phases where we statically analyze the code, find metadata and generate the manifest file cc @shuding @huozhi 

EDIT: 
- [Split page path utils](158fb00) After seeing one of the utils was being used by the client while it was defined originally in the server, with this PR we are splitting the util into multiple files and moving it to `shared/lib` in order to make explicit that those can be also imported from client.

packages/next/build/entries.ts

@@ -15,12 +15,11 @@ import { EDGE_RUNTIME_WEBPACK } from '../shared/lib/constants'
 import { MIDDLEWARE_ROUTE } from '../lib/constants'
 import { __ApiPreviewProps } from '../server/api-utils'
 import { isTargetLikeServerless } from '../server/utils'
-import { normalizePagePath } from '../server/normalize-page-path'
-import { normalizePathSep } from '../server/denormalize-page-path'
-import { ssrEntries } from './webpack/plugins/middleware-plugin'
 import { warn } from './output/log'
 import { parse } from '../build/swc'
 import { isFlightPage, withoutRSCExtensions } from './utils'
+import { normalizePathSep } from '../shared/lib/page-path/normalize-path-sep'
+import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path'
 
 type ObjectValue<T> = T extends { [key: string]: infer V } ? V : never
 
@@ -233,7 +232,6 @@ export function getEdgeServerEntry(opts: {
   isDev: boolean
   page: string
   pages: { [page: string]: string }
-  ssrEntries: Map<string, { requireFlightManifest: boolean }>
 }): ObjectValue<webpack5.EntryObject> {
   if (opts.page.match(MIDDLEWARE_ROUTE)) {
     const loaderParams: MiddlewareLoaderOptions = {
@@ -258,10 +256,6 @@ export function getEdgeServerEntry(opts: {
     stringifiedConfig: JSON.stringify(opts.config),
   }
 
-  ssrEntries.set(opts.bundlePath, {
-    requireFlightManifest: isFlightPage(opts.config, opts.absolutePagePath),
-  })
-
   return `next-middleware-ssr-loader?${stringify(loaderParams)}!`
 }
 
@@ -375,7 +369,6 @@ export async function createEntrypoints(params: CreateEntrypointsParams) {
             bundlePath: clientBundlePath,
             isDev: false,
             page,
-            ssrEntries,
           })
         },
       })

packages/next/build/index.ts

@@ -63,7 +63,7 @@ import { __ApiPreviewProps } from '../server/api-utils'
 import loadConfig from '../server/config'
 import { isTargetLikeServerless } from '../server/utils'
 import { BuildManifest } from '../server/get-page-files'
-import { normalizePagePath } from '../server/normalize-page-path'
+import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path'
 import { getPagePath } from '../server/require'
 import * as ciEnvironment from '../telemetry/ci-info'
 import {

packages/next/build/utils.ts

@@ -26,10 +26,6 @@ import { isDynamicRoute } from '../shared/lib/router/utils/is-dynamic'
 import escapePathDelimiters from '../shared/lib/router/utils/escape-path-delimiters'
 import { findPageFile } from '../server/lib/find-page-file'
 import { GetStaticPaths, PageConfig } from 'next/types'
-import {
-  denormalizePagePath,
-  normalizePagePath,
-} from '../server/normalize-page-path'
 import { BuildManifest } from '../server/get-page-files'
 import { removePathTrailingSlash } from '../client/normalize-trailing-slash'
 import { UnwrapPromise } from '../lib/coalesced-function'
@@ -42,6 +38,8 @@ import isError from '../lib/is-error'
 import { recursiveDelete } from '../lib/recursive-delete'
 import { Sema } from 'next/dist/compiled/async-sema'
 import { MiddlewareManifest } from './webpack/plugins/middleware-plugin'
+import { denormalizePagePath } from '../shared/lib/page-path/denormalize-page-path'
+import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path'
 
 const { builtinModules } = require('module')
 const RESERVED_PAGE = /^\/(_app|_error|_document|api(\/|$))/

packages/next/build/webpack/loaders/get-module-build-info.ts

@@ -0,0 +1,29 @@
+import { webpack5 } from 'next/dist/compiled/webpack/webpack'
+
+/**
+ * A getter for module build info that casts to the type it should have.
+ * We also expose here types to make easier to use it.
+ */
+export function getModuleBuildInfo(webpackModule: webpack5.Module) {
+  return webpackModule.buildInfo as {
+    nextEdgeMiddleware?: EdgeMiddlewareMeta
+    nextEdgeSSR?: EdgeSSRMeta
+    nextUsedEnvVars?: Set<string>
+    nextWasmMiddlewareBinding?: WasmBinding
+    usingIndirectEval?: boolean | Set<string>
+  }
+}
+
+export interface EdgeMiddlewareMeta {
+  page: string
+}
+
+export interface EdgeSSRMeta {
+  isServerComponent: boolean
+  page: string
+}
+
+export interface WasmBinding {
+  filePath: string
+  name: string
+}

packages/next/build/webpack/loaders/next-middleware-loader.ts

@@ -1,3 +1,4 @@
+import { getModuleBuildInfo } from './get-module-build-info'
 import { stringifyRequest } from '../stringify-request'
 
 export type MiddlewareLoaderOptions = {
@@ -8,6 +9,10 @@ export type MiddlewareLoaderOptions = {
 export default function middlewareLoader(this: any) {
   const { absolutePagePath, page }: MiddlewareLoaderOptions = this.getOptions()
   const stringifiedPagePath = stringifyRequest(this, absolutePagePath)
+  const buildInfo = getModuleBuildInfo(this._module)
+  buildInfo.nextEdgeMiddleware = {
+    page: page.replace(/\/_middleware$/, '') || '/',
+  }
 
   return `
         import { adapter } from 'next/dist/server/web/adapter'

packages/next/build/webpack/loaders/next-middleware-ssr-loader/index.ts

@@ -1,3 +1,4 @@
+import { getModuleBuildInfo } from '../get-module-build-info'
 import { stringifyRequest } from '../../stringify-request'
 
 export type MiddlewareSSRLoaderQuery = {
@@ -27,7 +28,13 @@ export default async function middlewareSSRLoader(this: any) {
     absoluteErrorPath,
     isServerComponent,
     stringifiedConfig,
-  }: MiddlewareSSRLoaderQuery = this.getOptions()
+  } = this.getOptions()
+
+  const buildInfo = getModuleBuildInfo(this._module)
+  buildInfo.nextEdgeSSR = {
+    isServerComponent: isServerComponent === 'true',
+    page: page,
+  }
 
   const stringifiedPagePath = stringifyRequest(this, absolutePagePath)
   const stringifiedAppPath = stringifyRequest(this, absoluteAppPath)

packages/next/build/webpack/loaders/next-middleware-wasm-loader.ts

@@ -1,15 +1,11 @@
+import { getModuleBuildInfo } from './get-module-build-info'
 import crypto from 'crypto'
 
-export type WasmBinding = {
-  filePath: string
-  name: string
-}
-
 export default function MiddlewareWasmLoader(this: any, source: Buffer) {
   const name = `wasm_${sha1(source)}`
   const filePath = `edge-chunks/${name}.wasm`
-  const binding: WasmBinding = { filePath: `server/${filePath}`, name }
-  this._module.buildInfo.nextWasmMiddlewareBinding = binding
+  const buildInfo = getModuleBuildInfo(this._module)
+  buildInfo.nextWasmMiddlewareBinding = { filePath: `server/${filePath}`, name }
   this.emitFile(`/${filePath}`, source, null)
   return `module.exports = ${name};`
 }

packages/next/build/webpack/loaders/next-serverless-loader/page-handler.ts

@@ -6,7 +6,7 @@ import { getUtils, vercelHeader, ServerlessHandlerCtx } from './utils'
 
 import { renderToHTML } from '../../../../server/render'
 import { tryGetPreviewData } from '../../../../server/api-utils/node'
-import { denormalizePagePath } from '../../../../server/denormalize-page-path'
+import { denormalizePagePath } from '../../../../shared/lib/page-path/denormalize-page-path'
 import { setLazyProp, getCookieParser } from '../../../../server/api-utils'
 import { getRedirectStatus } from '../../../../lib/load-custom-routes'
 import getRouteNoAssetPath from '../../../../shared/lib/router/utils/get-route-from-asset-path'

packages/next/build/webpack/loaders/next-serverless-loader/utils.ts

@@ -23,7 +23,7 @@ import { __ApiPreviewProps } from '../../../../server/api-utils'
 import { acceptLanguage } from '../../../../server/accept-header'
 import { detectLocaleCookie } from '../../../../shared/lib/i18n/detect-locale-cookie'
 import { detectDomainLocale } from '../../../../shared/lib/i18n/detect-domain-locale'
-import { denormalizePagePath } from '../../../../server/denormalize-page-path'
+import { denormalizePagePath } from '../../../../shared/lib/page-path/denormalize-page-path'
 import cookie from 'next/dist/compiled/cookie'
 import { TEMPORARY_REDIRECT_STATUS } from '../../../../shared/lib/constants'
 import { addRequestMeta } from '../../../../server/request-meta'