From 5a371909e40ee4ca8d7fd68953349ca4c7271748 Mon Sep 17 00:00:00 2001 From: Will Temple Date: Tue, 24 Dec 2024 15:27:13 -0500 Subject: [PATCH 1/3] [hsj] Rework multipart backend and add support for new-style multipart requests. --- .../generated-defs/helpers/header.ts | 83 ++++++ .../generated-defs/helpers/index.ts | 2 + .../generated-defs/helpers/multipart.ts | 252 ++++++++++++++++++ .../generated-defs/helpers/router.ts | 103 ++++--- packages/http-server-javascript/package.json | 11 +- .../src/common/model.ts | 34 ++- packages/http-server-javascript/src/ctx.ts | 15 ++ .../src/helpers/header.ts | 55 ++++ .../src/helpers/multipart.ts | 226 ++++++++++++++++ .../src/helpers/router.ts | 103 ++++--- .../src/http/server/index.ts | 162 +++++------ .../src/http/server/multipart.ts | 247 +++++++++++++++++ .../src/http/server/router.ts | 102 ++++--- packages/http-server-javascript/src/index.ts | 6 +- packages/http-server-javascript/src/lib.ts | 6 + .../http-server-javascript/src/util/name.ts | 4 +- .../test/header.test.ts | 26 ++ .../test/multipart.test.ts | 169 ++++++++++++ .../http-server-javascript/vitest.config.ts | 4 + pnpm-lock.yaml | 9 + 20 files changed, 1387 insertions(+), 232 deletions(-) create mode 100644 packages/http-server-javascript/generated-defs/helpers/header.ts create mode 100644 packages/http-server-javascript/generated-defs/helpers/multipart.ts create mode 100644 packages/http-server-javascript/src/helpers/header.ts create mode 100644 packages/http-server-javascript/src/helpers/multipart.ts create mode 100644 packages/http-server-javascript/src/http/server/multipart.ts create mode 100644 packages/http-server-javascript/test/header.test.ts create mode 100644 packages/http-server-javascript/test/multipart.test.ts create mode 100644 packages/http-server-javascript/vitest.config.ts diff --git a/packages/http-server-javascript/generated-defs/helpers/header.ts b/packages/http-server-javascript/generated-defs/helpers/header.ts new file mode 100644 index 00000000000..442c6d83e74 --- /dev/null +++ b/packages/http-server-javascript/generated-defs/helpers/header.ts @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation +// Licensed under the MIT license. + +import { Module } from "../../src/ctx.js"; + +export let module: Module = undefined as any; + +// prettier-ignore +const lines = [ + "// Copyright (c) Microsoft Corporation", + "// Licensed under the MIT license.", + "", + "export interface HeaderValueParameters {", + " value: string;", + " verbatim: string;", + " params: { [k: string]: string };", + "}", + "", + "/**", + " * Parses a header value that may contain additional parameters (e.g. `text/html; charset=utf-8`).", + " * @param headerValueText - the text of the header value to parse", + " * @returns an object containing the value and a map of parameters", + " */", + "export function parseHeaderValueParameters
(", + " headerValueText: Header,", + "): undefined extends Header ? HeaderValueParameters | undefined : HeaderValueParameters {", + " if (headerValueText === undefined) {", + " return undefined as any;", + " }", + "", + " const idx = headerValueText.indexOf(\";\");", + " const [value, _paramsText] =", + " idx === -1", + " ? [headerValueText, \"\"]", + " : [headerValueText.slice(0, idx), headerValueText.slice(idx + 1)];", + "", + " let paramsText = _paramsText;", + "", + " // Parameters are a sequence of key=value pairs separated by semicolons, but the value may be quoted in which case it", + " // may contain semicolons. We use a regular expression to iteratively split the parameters into key=value pairs.", + " const params: { [k: string]: string } = {};", + "", + " let match;", + "", + " // TODO: may need to support ext-parameter (e.g. \"filename*=UTF-8''%e2%82%ac%20rates\" => { filename: \"€ rates\" }).", + " // By default we decoded everything as UTF-8, and non-UTF-8 agents are a dying breed, but we may need to support", + " // this for completeness. If we do support it, we'll prefer an ext-parameter over a regular parameter. Currently, we'll", + " // just treat them as separate keys and put the raw value in the parameter.", + " //", + " // https://datatracker.ietf.org/doc/html/rfc5987#section-3.2.1", + " while ((match = paramsText.match(/\\s*([^=]+)=(?:\"([^\"]+)\"|([^;]+));?/))) {", + " const [, key, quotedValue, unquotedValue] = match;", + "", + " params[key.trim()] = quotedValue ?? unquotedValue;", + "", + " paramsText = paramsText.slice(match[0].length);", + " }", + "", + " return {", + " value: value.trim(),", + " verbatim: headerValueText,", + " params,", + " };", + "}", + "", +]; + +export async function createModule(parent: Module): Promise { + if (module) return module; + + module = { + name: "header", + cursor: parent.cursor.enter("header"), + imports: [], + declarations: [], + }; + + module.declarations.push(lines); + + parent.declarations.push(module); + + return module; +} diff --git a/packages/http-server-javascript/generated-defs/helpers/index.ts b/packages/http-server-javascript/generated-defs/helpers/index.ts index 3876a28a17f..1db2f0741ae 100644 --- a/packages/http-server-javascript/generated-defs/helpers/index.ts +++ b/packages/http-server-javascript/generated-defs/helpers/index.ts @@ -16,6 +16,8 @@ export async function createModule(parent: Module): Promise { }; // Child modules + await import("./header.js").then((m) => m.createModule(module)); + await import("./multipart.js").then((m) => m.createModule(module)); await import("./router.js").then((m) => m.createModule(module)); parent.declarations.push(module); diff --git a/packages/http-server-javascript/generated-defs/helpers/multipart.ts b/packages/http-server-javascript/generated-defs/helpers/multipart.ts new file mode 100644 index 00000000000..edb37e31409 --- /dev/null +++ b/packages/http-server-javascript/generated-defs/helpers/multipart.ts @@ -0,0 +1,252 @@ +// Copyright (c) Microsoft Corporation +// Licensed under the MIT license. + +import { Module } from "../../src/ctx.js"; + +export let module: Module = undefined as any; + +// prettier-ignore +const lines = [ + "// Copyright (c) Microsoft Corporation", + "// Licensed under the MIT license.", + "", + "import type * as http from \"node:http\";", + "", + "export interface HttpPart {", + " headers: { [k: string]: string | undefined };", + " body: ReadableStream;", + "}", + "", + "/**", + " * Consumes a stream of incoming data and splits it into individual streams for each part of a multipart request, using", + " * the provided `boundary` value.", + " */", + "function MultipartBoundaryTransformStream(", + " boundary: string,", + "): ReadableWritablePair, Buffer> {", + " let buffer: Buffer = Buffer.alloc(0);", + " // Initialize subcontroller to an object that does nothing. Multipart bodies may contain a preamble before the first", + " // boundary, so this dummy controller will discard it.", + " let subController: { enqueue(chunk: Buffer): void; close(): void } | null = {", + " enqueue() {},", + " close() {},", + " };", + "", + " let boundarySplit = Buffer.from(`--${boundary}`);", + " let initialized = false;", + "", + " const bufferKeepLength = boundarySplit.length + BUF_CRLFCRLF.length - 1;", + " let _readableController: ReadableStreamDefaultController> = null as any;", + "", + " const readable = new ReadableStream>({", + " start(controller) {", + " _readableController = controller;", + " },", + " });", + "", + " const readableController = _readableController;", + "", + " const writable = new WritableStream({", + " write: async (chunk) => {", + " buffer = Buffer.concat([buffer, chunk]);", + "", + " const index = buffer.indexOf(boundarySplit);", + "", + " if (index !== -1) {", + " // We found a boundary, emit everything before it and initialize a new stream for the next part.", + "", + " // We are initialized if we have consumed at least one chunk.", + " //", + " // Cases", + " // 1. If the index is zero and we aren't initialized, there was no preamble.", + " // 2. If the index is zero and we are initialized, then we had to have found \\r\\n--boundary, nothing special to do.", + " // 3. If the index is not zero, and we are initialized, then we found \\r\\n--boundary somewhere in the middle,", + " // nothing special to do.", + " // 4. If the index is not zero and we aren't initialized, then we need to check that boundarySplit was preceded", + " // by \\r\\n for validity, because the preamble must end with \\r\\n.", + "", + " if (index > 0) {", + " if (!initialized) {", + " if (!buffer.subarray(index - 2, index).equals(Buffer.from(\"\\r\\n\"))) {", + " readableController.error(new Error(\"Invalid preamble in multipart body.\"));", + " } else {", + " await enqueueSub(buffer.subarray(0, index - 2));", + " }", + " } else {", + " await enqueueSub(buffer.subarray(0, index));", + " }", + " }", + "", + " // We enqueued everything before the boundary, so we clear the buffer past the boundary", + " buffer = buffer.subarray(index + boundarySplit.length);", + "", + " // We're done with the current part, so close the stream. If this is the opening boundary, there won't be a", + " // subcontroller yet.", + " subController?.close();", + " subController = null;", + " }", + "", + " if (buffer.length > bufferKeepLength) {", + " await enqueueSub(buffer.subarray(0, -bufferKeepLength));", + " buffer = buffer.subarray(-bufferKeepLength);", + " }", + "", + " if (!initialized) {", + " initialized = true;", + " boundarySplit = Buffer.from(`\\r\\n${boundarySplit}`);", + " }", + " },", + " close() {", + " if (buffer.toString(\"utf-8\") !== \"--\") {", + " readableController.error(new Error(\"Multipart body terminated unexpectedly.\"));", + " }", + "", + " readableController.close();", + " },", + " });", + "", + " async function enqueueSub(s: Buffer) {", + " subController ??= await new Promise((resolve) => {", + " readableController.enqueue(", + " new ReadableStream({", + " start: (controller) => resolve(controller),", + " }),", + " );", + " });", + "", + " subController.enqueue(s);", + " }", + "", + " return { readable, writable };", + "}", + "", + "const BUF_CRLFCRLF = Buffer.from(\"\\r\\n\\r\\n\");", + "", + "/**", + " * Consumes a stream of the contents of a single part of a multipart request and emits an `HttpPart` object for each part.", + " * This consumes just enough of the stream to read the headers, and then forwards the rest of the stream as the body.", + " */", + "class HttpPartTransform extends TransformStream, HttpPart> {", + " constructor() {", + " super({", + " transform: async (partRaw, controller) => {", + " const reader = partRaw.getReader();", + "", + " let buf = Buffer.alloc(0);", + " let idx;", + "", + " while ((idx = buf.indexOf(BUF_CRLFCRLF)) === -1) {", + " const { done, value } = await reader.read();", + " if (done) {", + " throw new Error(\"Unexpected end of part.\");", + " }", + " buf = Buffer.concat([buf, value]);", + " }", + "", + " const headerText = buf.subarray(0, idx).toString(\"utf-8\").trim();", + "", + " const headers = Object.fromEntries(", + " headerText.split(\"\\r\\n\").map((line) => {", + " const [name, value] = line.split(\": \", 2);", + "", + " return [name.toLowerCase(), value];", + " }),", + " ) as { [k: string]: string };", + "", + " const body = new ReadableStream({", + " start(controller) {", + " controller.enqueue(buf.subarray(idx + BUF_CRLFCRLF.length));", + " },", + " async pull(controller) {", + " const { done, value } = await reader.read();", + "", + " if (done) {", + " controller.close();", + " } else {", + " controller.enqueue(value);", + " }", + " },", + " });", + "", + " controller.enqueue({ headers, body });", + " },", + " });", + " }", + "}", + "", + "/**", + " * Processes a request as a multipart request, returning a stream of `HttpPart` objects, each representing an individual", + " * part in the multipart request.", + " *", + " * Only call this function if you have already validated the content type of the request and confirmed that it is a", + " * multipart request.", + " *", + " * @throws Error if the content-type header is missing or does not contain a boundary field.", + " *", + " * @param request - the incoming request to parse as multipart", + " * @returns a stream of HttpPart objects, each representing an individual part in the multipart request", + " */", + "export function createMultipartReadable(request: http.IncomingMessage): ReadableStream {", + " const boundary = request.headers[\"content-type\"]", + " ?.split(\";\")", + " .find((s) => s.includes(\"boundary=\"))", + " ?.split(\"=\", 2)[1];", + " if (!boundary) {", + " throw new Error(\"Invalid request: missing boundary in content-type.\");", + " }", + "", + " const bodyStream = new ReadableStream({", + " start(controller) {", + " request.on(\"data\", (chunk: Buffer) => {", + " controller.enqueue(chunk);", + " });", + " request.on(\"end\", () => controller.close());", + " },", + " });", + "", + " return bodyStream", + " .pipeThrough(MultipartBoundaryTransformStream(boundary))", + " .pipeThrough(new HttpPartTransform());", + "}", + "", + "// Gross polyfill because Safari doesn't support this yet.", + "//", + "// https://bugs.webkit.org/show_bug.cgi?id=194379", + "// https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream#browser_compatibility", + "(ReadableStream.prototype as any)[Symbol.asyncIterator] ??= async function* () {", + " const reader = this.getReader();", + " try {", + " while (true) {", + " const { done, value } = await reader.read();", + " if (done) return value;", + " yield value;", + " }", + " } finally {", + " reader.releaseLock();", + " }", + "};", + "", + "declare global {", + " interface ReadableStream {", + " [Symbol.asyncIterator](): AsyncIterableIterator;", + " }", + "}", + "", +]; + +export async function createModule(parent: Module): Promise { + if (module) return module; + + module = { + name: "multipart", + cursor: parent.cursor.enter("multipart"), + imports: [], + declarations: [], + }; + + module.declarations.push(lines); + + parent.declarations.push(module); + + return module; +} diff --git a/packages/http-server-javascript/generated-defs/helpers/router.ts b/packages/http-server-javascript/generated-defs/helpers/router.ts index 2430a8198d7..12ac11cc4c7 100644 --- a/packages/http-server-javascript/generated-defs/helpers/router.ts +++ b/packages/http-server-javascript/generated-defs/helpers/router.ts @@ -31,11 +31,7 @@ const lines = [ " * @param response - The outgoing HTTP response.", " * @param next - Calls the next policy in the chain.", " */", - " (", - " request: http.IncomingMessage,", - " response: http.ServerResponse,", - " next: (request?: http.IncomingMessage) => void,", - " ): void;", + " (ctx: HttpContext, next: (request?: http.IncomingMessage) => void): void;", "}", "", "/**", @@ -47,43 +43,36 @@ const lines = [ " * @param policies - The policies to apply to the request.", " * @param out - The function to call after the policies have been applied.", " */", - "export function createPolicyChain<", - " Out extends (", - " ctx: HttpContext,", - " request: http.IncomingMessage,", - " response: http.ServerResponse,", - " ...rest: any[]", - " ) => void,", - ">(name: string, policies: Policy[], out: Out): Out {", + "export function createPolicyChain void>(", + " name: string,", + " policies: Policy[],", + " out: Out,", + "): Out {", " let outParams: any[];", " if (policies.length === 0) {", " return out;", " }", "", - " function applyPolicy(", - " ctx: HttpContext,", - " request: http.IncomingMessage,", - " response: http.ServerResponse,", - " index: number,", - " ) {", + " function applyPolicy(ctx: HttpContext, index: number) {", " if (index >= policies.length) {", - " return out(ctx, request, response, ...outParams);", + " return out(ctx, ...outParams);", " }", "", - " policies[index](request, response, function nextPolicy(nextRequest) {", - " applyPolicy(ctx, nextRequest ?? request, response, index + 1);", + " policies[index](ctx, function nextPolicy(nextRequest) {", + " applyPolicy(", + " {", + " ...ctx,", + " request: nextRequest ?? ctx.request,", + " },", + " index + 1,", + " );", " });", " }", "", " return {", - " [name](", - " ctx: HttpContext,", - " request: http.IncomingMessage,", - " response: http.ServerResponse,", - " ...params: any[]", - " ) {", + " [name](ctx: HttpContext, ...params: any[]) {", " outParams = params;", - " applyPolicy(ctx, request, response, 0);", + " applyPolicy(ctx, 0);", " },", " }[name] as Out;", "}", @@ -122,12 +111,7 @@ const lines = [ "export function createPolicyChainForRoute<", " RouteConfig extends { [k: string]: object },", " InterfaceName extends keyof RouteConfig,", - " Out extends (", - " ctx: HttpContext,", - " request: http.IncomingMessage,", - " response: http.ServerResponse,", - " ...rest: any[]", - " ) => void,", + " Out extends (ctx: HttpContext, ...rest: any[]) => void,", ">(", " name: string,", " routePolicies: RoutePolicies,", @@ -188,7 +172,9 @@ const lines = [ " routePolicies?: RoutePolicies;", "", " /**", - " * A handler for requests that do not match any known route and method.", + " * A handler for requests where the resource is not found.", + " *", + " * The router will call this function when no route matches the incoming request.", " *", " * If this handler is not provided, a 404 Not Found response with a text body will be returned.", " *", @@ -197,10 +183,9 @@ const lines = [ " * This handler is unreachable when using the Express middleware, as it will forward non-matching requests to the", " * next middleware layer in the stack.", " *", - " * @param request - The incoming HTTP request.", - " * @param response - The outgoing HTTP response.", + " * @param ctx - The HTTP context for the request.", " */", - " onRequestNotFound?: (request: http.IncomingMessage, response: http.ServerResponse) => void;", + " onRequestNotFound?: (ctx: HttpContext) => void;", "", " /**", " * A handler for requests that fail to validate inputs.", @@ -210,17 +195,11 @@ const lines = [ " *", " * You _MUST_ call `response.end()` to terminate the response.", " *", - " * @param request - The incoming HTTP request.", - " * @param response - The outgoing HTTP response.", + " * @param ctx - The HTTP context for the request.", " * @param route - The route that was matched.", " * @param error - The validation error that was thrown.", " */", - " onInvalidRequest?: (", - " request: http.IncomingMessage,", - " response: http.ServerResponse,", - " route: string,", - " error: ValidationError,", - " ) => void;", + " onInvalidRequest?: (ctx: HttpContext, route: string, error: ValidationError) => void;", "", " /**", " * A handler for requests that throw an error during processing.", @@ -232,15 +211,10 @@ const lines = [ " *", " * If this handler itself throws an Error, the router will respond with a 500 Internal Server Error", " *", + " * @param ctx - The HTTP context for the request.", " * @param error - The error that was thrown.", - " * @param request - The incoming HTTP request.", - " * @param response - The outgoing HTTP response.", " */", - " onInternalError?(", - " error: unknown,", - " request: http.IncomingMessage,", - " response: http.ServerResponse,", - " ): void;", + " onInternalError?(ctx: HttpContext, error: Error): void;", "}", "", "/** Context information for operations carried over the HTTP protocol. */", @@ -249,6 +223,27 @@ const lines = [ " request: http.IncomingMessage;", " /** The outgoing response object. */", " response: http.ServerResponse;", + "", + " /**", + " * Error handling functions provided by the HTTP router. Service implementations may call these methods in case a", + " * resource is not found, a request is invalid, or an internal error occurs.", + " *", + " * These methods will respond to the client with the appropriate status code and message.", + " */", + " errorHandlers: {", + " /**", + " * Signals that the requested resource was not found.", + " */", + " onRequestNotFound: Exclude;", + " /**", + " * Signals that the request was invalid.", + " */", + " onInvalidRequest: Exclude;", + " /**", + " * Signals that an internal error occurred.", + " */", + " onInternalError: Exclude;", + " };", "}", "", ]; diff --git a/packages/http-server-javascript/package.json b/packages/http-server-javascript/package.json index 6cb073d5611..921ad5f0096 100644 --- a/packages/http-server-javascript/package.json +++ b/packages/http-server-javascript/package.json @@ -32,8 +32,10 @@ "build:src": "tsc -p ./tsconfig.json", "build:helpers": "tsx ./build-helpers.ts", "watch": "tsc -p . --watch", - "test": "echo No tests specified", - "test:ci": "echo No tests specified", + "test": "vitest run", + "test:watch": "vitest -w", + "test:ui": "vitest --ui", + "test:ci": "vitest run --coverage --reporter=junit --reporter=default", "lint": "eslint . --max-warnings=0", "lint:fix": "eslint . --fix", "regen-docs": "echo Doc generation disabled for this package." @@ -49,7 +51,10 @@ "@types/node": "~22.7.9", "@typespec/compiler": "workspace:~", "@typespec/http": "workspace:~", + "@vitest/coverage-v8": "^2.1.5", + "@vitest/ui": "^2.1.2", "tsx": "^4.19.2", - "typescript": "~5.6.3" + "typescript": "~5.6.3", + "vitest": "^2.1.5" } } diff --git a/packages/http-server-javascript/src/common/model.ts b/packages/http-server-javascript/src/common/model.ts index bf0a01234d9..3b0ac93cb1b 100644 --- a/packages/http-server-javascript/src/common/model.ts +++ b/packages/http-server-javascript/src/common/model.ts @@ -88,14 +88,20 @@ export function* emitModel( } export function emitModelLiteral(ctx: JsContext, model: Model, module: Module): string { - const properties = [...model.properties.values()].map((prop) => { - const nameCase = parseCase(prop.name); - const questionMark = prop.optional ? "?" : ""; + const properties = [...model.properties.values()] + .map((prop) => { + if (isUnspeakable(prop.name)) { + return undefined; + } - const name = KEYWORDS.has(nameCase.camelCase) ? `_${nameCase.camelCase}` : nameCase.camelCase; + const nameCase = parseCase(prop.name); + const questionMark = prop.optional ? "?" : ""; - return `${name}${questionMark}: ${emitTypeReference(ctx, prop.type, prop, module)}`; - }); + const name = KEYWORDS.has(nameCase.camelCase) ? `_${nameCase.camelCase}` : nameCase.camelCase; + + return `${name}${questionMark}: ${emitTypeReference(ctx, prop.type, prop, module)}`; + }) + .filter((p) => !!p); return `{ ${properties.join("; ")} }`; } @@ -105,7 +111,7 @@ export function emitModelLiteral(ctx: JsContext, model: Model, module: Module): */ export function isWellKnownModel(ctx: JsContext, type: Model): boolean { const fullName = getFullyQualifiedTypeName(type); - return fullName === "TypeSpec.Record" || fullName === "TypeSpec.Array"; + return ["TypeSpec.Record", "TypeSpec.Array", "TypeSpec.Http.HttpPart"].includes(fullName); } /** @@ -122,20 +128,32 @@ export function emitWellKnownModel( module: Module, preferredAlternativeName?: string, ): string { - const arg = type.indexer!.value; switch (type.name) { case "Record": { + const arg = type.indexer!.value; return `{ [k: string]: ${emitTypeReference(ctx, arg, type, module, { altName: preferredAlternativeName && getRecordValueName(preferredAlternativeName), })} }`; } case "Array": { + const arg = type.indexer!.value; return asArrayType( emitTypeReference(ctx, arg, type, module, { altName: preferredAlternativeName && getArrayElementName(preferredAlternativeName), }), ); } + case "HttpPart": { + const argument = type.templateMapper!.args[0]; + + if (!(argument.entityKind === "Type" && argument.kind === "Model")) { + throw new Error("UNREACHABLE: HttpPart must have a Model argument"); + } + + return emitTypeReference(ctx, argument, type, module, { + altName: preferredAlternativeName && `${preferredAlternativeName}HttpPart`, + }); + } default: throw new Error(`UNREACHABLE: ${type.name}`); } diff --git a/packages/http-server-javascript/src/ctx.ts b/packages/http-server-javascript/src/ctx.ts index 02ab1559239..5b4a14b4b1c 100644 --- a/packages/http-server-javascript/src/ctx.ts +++ b/packages/http-server-javascript/src/ctx.ts @@ -117,6 +117,8 @@ export interface JsContext { * A map of all types that require serialization code to the formats they require. */ serializations: OnceQueue; + + gensym: (name: string) => string; } /** @@ -384,3 +386,16 @@ function getCommonPrefix(a: string[], b: string[]): string[] { return prefix; } + +const SYM_TAB = new WeakMap(); + +export function gensym(ctx: JsContext, name: string): string { + let symTab = SYM_TAB.get(ctx.program); + + if (symTab === undefined) { + symTab = { idx: 0 }; + SYM_TAB.set(ctx.program, symTab); + } + + return `__${name}_${symTab.idx++}`; +} diff --git a/packages/http-server-javascript/src/helpers/header.ts b/packages/http-server-javascript/src/helpers/header.ts new file mode 100644 index 00000000000..b177eb57be1 --- /dev/null +++ b/packages/http-server-javascript/src/helpers/header.ts @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation +// Licensed under the MIT license. + +export interface HeaderValueParameters { + value: string; + verbatim: string; + params: { [k: string]: string }; +} + +/** + * Parses a header value that may contain additional parameters (e.g. `text/html; charset=utf-8`). + * @param headerValueText - the text of the header value to parse + * @returns an object containing the value and a map of parameters + */ +export function parseHeaderValueParameters
( + headerValueText: Header, +): undefined extends Header ? HeaderValueParameters | undefined : HeaderValueParameters { + if (headerValueText === undefined) { + return undefined as any; + } + + const idx = headerValueText.indexOf(";"); + const [value, _paramsText] = + idx === -1 + ? [headerValueText, ""] + : [headerValueText.slice(0, idx), headerValueText.slice(idx + 1)]; + + let paramsText = _paramsText; + + // Parameters are a sequence of key=value pairs separated by semicolons, but the value may be quoted in which case it + // may contain semicolons. We use a regular expression to iteratively split the parameters into key=value pairs. + const params: { [k: string]: string } = {}; + + let match; + + // TODO: may need to support ext-parameter (e.g. "filename*=UTF-8''%e2%82%ac%20rates" => { filename: "€ rates" }). + // By default we decoded everything as UTF-8, and non-UTF-8 agents are a dying breed, but we may need to support + // this for completeness. If we do support it, we'll prefer an ext-parameter over a regular parameter. Currently, we'll + // just treat them as separate keys and put the raw value in the parameter. + // + // https://datatracker.ietf.org/doc/html/rfc5987#section-3.2.1 + while ((match = paramsText.match(/\s*([^=]+)=(?:"([^"]+)"|([^;]+));?/))) { + const [, key, quotedValue, unquotedValue] = match; + + params[key.trim()] = quotedValue ?? unquotedValue; + + paramsText = paramsText.slice(match[0].length); + } + + return { + value: value.trim(), + verbatim: headerValueText, + params, + }; +} diff --git a/packages/http-server-javascript/src/helpers/multipart.ts b/packages/http-server-javascript/src/helpers/multipart.ts new file mode 100644 index 00000000000..02362978a8f --- /dev/null +++ b/packages/http-server-javascript/src/helpers/multipart.ts @@ -0,0 +1,226 @@ +// Copyright (c) Microsoft Corporation +// Licensed under the MIT license. + +import type * as http from "node:http"; + +export interface HttpPart { + headers: { [k: string]: string | undefined }; + body: ReadableStream; +} + +/** + * Consumes a stream of incoming data and splits it into individual streams for each part of a multipart request, using + * the provided `boundary` value. + */ +function MultipartBoundaryTransformStream( + boundary: string, +): ReadableWritablePair, Buffer> { + let buffer: Buffer = Buffer.alloc(0); + // Initialize subcontroller to an object that does nothing. Multipart bodies may contain a preamble before the first + // boundary, so this dummy controller will discard it. + let subController: { enqueue(chunk: Buffer): void; close(): void } | null = { + enqueue() {}, + close() {}, + }; + + let boundarySplit = Buffer.from(`--${boundary}`); + let initialized = false; + + const bufferKeepLength = boundarySplit.length + BUF_CRLFCRLF.length - 1; + let _readableController: ReadableStreamDefaultController> = null as any; + + const readable = new ReadableStream>({ + start(controller) { + _readableController = controller; + }, + }); + + const readableController = _readableController; + + const writable = new WritableStream({ + write: async (chunk) => { + buffer = Buffer.concat([buffer, chunk]); + + const index = buffer.indexOf(boundarySplit); + + if (index !== -1) { + // We found a boundary, emit everything before it and initialize a new stream for the next part. + + // We are initialized if we have found the boundary at least once. + // + // Cases + // 1. If the index is zero and we aren't initialized, there was no preamble. + // 2. If the index is zero and we are initialized, then we had to have found \r\n--boundary, nothing special to do. + // 3. If the index is not zero, and we are initialized, then we found \r\n--boundary somewhere in the middle, + // nothing special to do. + // 4. If the index is not zero and we aren't initialized, then we need to check that boundarySplit was preceded + // by \r\n for validity, because the preamble must end with \r\n. + + if (index > 0) { + if (!initialized) { + if (!buffer.subarray(index - 2, index).equals(Buffer.from("\r\n"))) { + readableController.error(new Error("Invalid preamble in multipart body.")); + } else { + await enqueueSub(buffer.subarray(0, index - 2)); + } + } else { + await enqueueSub(buffer.subarray(0, index)); + } + } + + // We enqueued everything before the boundary, so we clear the buffer past the boundary + buffer = buffer.subarray(index + boundarySplit.length); + + // We're done with the current part, so close the stream. If this is the opening boundary, there won't be a + // subcontroller yet. + subController?.close(); + subController = null; + + if (!initialized) { + initialized = true; + boundarySplit = Buffer.from(`\r\n${boundarySplit}`); + } + } + + if (buffer.length > bufferKeepLength) { + await enqueueSub(buffer.subarray(0, -bufferKeepLength)); + buffer = buffer.subarray(-bufferKeepLength); + } + }, + close() { + if (buffer.toString("utf-8") !== "--") { + readableController.error(new Error("Unexpected characters after final boundary.")); + } + + subController?.close(); + + readableController.close(); + }, + }); + + async function enqueueSub(s: Buffer) { + subController ??= await new Promise((resolve) => { + readableController.enqueue( + new ReadableStream({ + start: (controller) => resolve(controller), + }), + ); + }); + + subController.enqueue(s); + } + + return { readable, writable }; +} + +const BUF_CRLFCRLF = Buffer.from("\r\n\r\n"); + +/** + * Consumes a stream of the contents of a single part of a multipart request and emits an `HttpPart` object for each part. + * This consumes just enough of the stream to read the headers, and then forwards the rest of the stream as the body. + */ +class HttpPartTransform extends TransformStream, HttpPart> { + constructor() { + super({ + transform: async (partRaw, controller) => { + const reader = partRaw.getReader(); + + let buf = Buffer.alloc(0); + let idx; + + while ((idx = buf.indexOf(BUF_CRLFCRLF)) === -1) { + const { done, value } = await reader.read(); + if (done) { + throw new Error("Unexpected end of part."); + } + buf = Buffer.concat([buf, value]); + } + + const headerText = buf.subarray(0, idx).toString("utf-8").trim(); + + const headers = Object.fromEntries( + headerText.split("\r\n").map((line) => { + const [name, value] = line.split(": ", 2); + + return [name.toLowerCase(), value]; + }), + ) as { [k: string]: string }; + + const body = new ReadableStream({ + start(controller) { + controller.enqueue(buf.subarray(idx + BUF_CRLFCRLF.length)); + }, + async pull(controller) { + const { done, value } = await reader.read(); + + if (done) { + controller.close(); + } else { + controller.enqueue(value); + } + }, + }); + + controller.enqueue({ headers, body }); + }, + }); + } +} + +/** + * Processes a request as a multipart request, returning a stream of `HttpPart` objects, each representing an individual + * part in the multipart request. + * + * Only call this function if you have already validated the content type of the request and confirmed that it is a + * multipart request. + * + * @throws Error if the content-type header is missing or does not contain a boundary field. + * + * @param request - the incoming request to parse as multipart + * @returns a stream of HttpPart objects, each representing an individual part in the multipart request + */ +export function createMultipartReadable(request: http.IncomingMessage): ReadableStream { + const boundary = request.headers["content-type"] + ?.split(";") + .find((s) => s.includes("boundary=")) + ?.split("=", 2)[1]; + if (!boundary) { + throw new Error("Invalid request: missing boundary in content-type."); + } + + const bodyStream = new ReadableStream({ + start(controller) { + request.on("data", (chunk: Buffer) => { + controller.enqueue(chunk); + }); + request.on("end", () => controller.close()); + }, + }); + + return bodyStream + .pipeThrough(MultipartBoundaryTransformStream(boundary)) + .pipeThrough(new HttpPartTransform()); +} + +// Gross polyfill because Safari doesn't support this yet. +// +// https://bugs.webkit.org/show_bug.cgi?id=194379 +// https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream#browser_compatibility +(ReadableStream.prototype as any)[Symbol.asyncIterator] ??= async function* () { + const reader = this.getReader(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) return value; + yield value; + } + } finally { + reader.releaseLock(); + } +}; + +declare global { + interface ReadableStream { + [Symbol.asyncIterator](): AsyncIterableIterator; + } +} diff --git a/packages/http-server-javascript/src/helpers/router.ts b/packages/http-server-javascript/src/helpers/router.ts index 26d8e81cd46..e07d765266b 100644 --- a/packages/http-server-javascript/src/helpers/router.ts +++ b/packages/http-server-javascript/src/helpers/router.ts @@ -22,11 +22,7 @@ export interface Policy { * @param response - The outgoing HTTP response. * @param next - Calls the next policy in the chain. */ - ( - request: http.IncomingMessage, - response: http.ServerResponse, - next: (request?: http.IncomingMessage) => void, - ): void; + (ctx: HttpContext, next: (request?: http.IncomingMessage) => void): void; } /** @@ -38,43 +34,36 @@ export interface Policy { * @param policies - The policies to apply to the request. * @param out - The function to call after the policies have been applied. */ -export function createPolicyChain< - Out extends ( - ctx: HttpContext, - request: http.IncomingMessage, - response: http.ServerResponse, - ...rest: any[] - ) => void, ->(name: string, policies: Policy[], out: Out): Out { +export function createPolicyChain void>( + name: string, + policies: Policy[], + out: Out, +): Out { let outParams: any[]; if (policies.length === 0) { return out; } - function applyPolicy( - ctx: HttpContext, - request: http.IncomingMessage, - response: http.ServerResponse, - index: number, - ) { + function applyPolicy(ctx: HttpContext, index: number) { if (index >= policies.length) { - return out(ctx, request, response, ...outParams); + return out(ctx, ...outParams); } - policies[index](request, response, function nextPolicy(nextRequest) { - applyPolicy(ctx, nextRequest ?? request, response, index + 1); + policies[index](ctx, function nextPolicy(nextRequest) { + applyPolicy( + { + ...ctx, + request: nextRequest ?? ctx.request, + }, + index + 1, + ); }); } return { - [name]( - ctx: HttpContext, - request: http.IncomingMessage, - response: http.ServerResponse, - ...params: any[] - ) { + [name](ctx: HttpContext, ...params: any[]) { outParams = params; - applyPolicy(ctx, request, response, 0); + applyPolicy(ctx, 0); }, }[name] as Out; } @@ -113,12 +102,7 @@ export type RoutePolicies = { export function createPolicyChainForRoute< RouteConfig extends { [k: string]: object }, InterfaceName extends keyof RouteConfig, - Out extends ( - ctx: HttpContext, - request: http.IncomingMessage, - response: http.ServerResponse, - ...rest: any[] - ) => void, + Out extends (ctx: HttpContext, ...rest: any[]) => void, >( name: string, routePolicies: RoutePolicies, @@ -179,7 +163,9 @@ export interface RouterOptions< routePolicies?: RoutePolicies; /** - * A handler for requests that do not match any known route and method. + * A handler for requests where the resource is not found. + * + * The router will call this function when no route matches the incoming request. * * If this handler is not provided, a 404 Not Found response with a text body will be returned. * @@ -188,10 +174,9 @@ export interface RouterOptions< * This handler is unreachable when using the Express middleware, as it will forward non-matching requests to the * next middleware layer in the stack. * - * @param request - The incoming HTTP request. - * @param response - The outgoing HTTP response. + * @param ctx - The HTTP context for the request. */ - onRequestNotFound?: (request: http.IncomingMessage, response: http.ServerResponse) => void; + onRequestNotFound?: (ctx: HttpContext) => void; /** * A handler for requests that fail to validate inputs. @@ -201,17 +186,11 @@ export interface RouterOptions< * * You _MUST_ call `response.end()` to terminate the response. * - * @param request - The incoming HTTP request. - * @param response - The outgoing HTTP response. + * @param ctx - The HTTP context for the request. * @param route - The route that was matched. * @param error - The validation error that was thrown. */ - onInvalidRequest?: ( - request: http.IncomingMessage, - response: http.ServerResponse, - route: string, - error: ValidationError, - ) => void; + onInvalidRequest?: (ctx: HttpContext, route: string, error: ValidationError) => void; /** * A handler for requests that throw an error during processing. @@ -223,15 +202,10 @@ export interface RouterOptions< * * If this handler itself throws an Error, the router will respond with a 500 Internal Server Error * + * @param ctx - The HTTP context for the request. * @param error - The error that was thrown. - * @param request - The incoming HTTP request. - * @param response - The outgoing HTTP response. */ - onInternalError?( - error: unknown, - request: http.IncomingMessage, - response: http.ServerResponse, - ): void; + onInternalError?(ctx: HttpContext, error: Error): void; } /** Context information for operations carried over the HTTP protocol. */ @@ -240,4 +214,25 @@ export interface HttpContext { request: http.IncomingMessage; /** The outgoing response object. */ response: http.ServerResponse; + + /** + * Error handling functions provided by the HTTP router. Service implementations may call these methods in case a + * resource is not found, a request is invalid, or an internal error occurs. + * + * These methods will respond to the client with the appropriate status code and message. + */ + errorHandlers: { + /** + * Signals that the requested resource was not found. + */ + onRequestNotFound: Exclude; + /** + * Signals that the request was invalid. + */ + onInvalidRequest: Exclude; + /** + * Signals that an internal error occurred. + */ + onInternalError: Exclude; + }; } diff --git a/packages/http-server-javascript/src/http/server/index.ts b/packages/http-server-javascript/src/http/server/index.ts index 5918ef8caf5..d9b031d0985 100644 --- a/packages/http-server-javascript/src/http/server/index.ts +++ b/packages/http-server-javascript/src/http/server/index.ts @@ -29,6 +29,9 @@ import { HttpContext } from "../index.js"; import { module as routerHelpers } from "../../../generated-defs/helpers/router.js"; import { reportDiagnostic } from "../../lib.js"; import { differentiateUnion, writeCodeTree } from "../../util/differentiate.js"; +import { emitMultipart, emitMultipartLegacy } from "./multipart.js"; + +import { module as headerHelpers } from "../../../generated-defs/helpers/header.js"; const DEFAULT_CONTENT_TYPE = "application/json"; @@ -42,11 +45,6 @@ const DEFAULT_CONTENT_TYPE = "application/json"; export function emitRawServer(ctx: HttpContext, operationsModule: Module): Module { const serverRawModule = createModule("server-raw", operationsModule); - serverRawModule.imports.push({ - binder: "* as http", - from: "node:http", - }); - serverRawModule.imports.push({ binder: ["HttpContext"], from: routerHelpers, @@ -89,11 +87,16 @@ function* emitRawServerOperation( const functionName = keywordSafe(containerNameCase.snakeCase + "_" + operationNameCase.snakeCase); + const names: Names = { + ctx: ctx.gensym("ctx"), + result: ctx.gensym("result"), + operations: ctx.gensym("operations"), + queryParams: ctx.gensym("queryParams"), + }; + yield `export async function ${functionName}(`; - yield ` ctx: HttpContext,`; - yield ` request: http.IncomingMessage,`; - yield ` response: http.ServerResponse,`; - yield ` operations: ${containerNameCase.pascalCase},`; + yield ` ${names.ctx}: HttpContext,`; + yield ` ${names.operations}: ${containerNameCase.pascalCase},`; for (const pathParam of pathParameters) { yield ` ${parseCase(pathParam.param.name).camelCase}: string,`; @@ -114,7 +117,7 @@ function* emitRawServerOperation( parameter.param.type.kind === "ModelProperty" ? parameter.param.type : parameter.param; switch (parameter.type) { case "header": - yield* indent(emitHeaderParamBinding(ctx, parameter)); + yield* indent(emitHeaderParamBinding(ctx, operation, names, parameter)); break; case "cookie": throw new UnimplementedError("cookie parameters"); @@ -136,12 +139,12 @@ function* emitRawServerOperation( } if (queryParams.length > 0) { - yield ` const __query_params = new URLSearchParams(request.url!.split("?", 1)[1] ?? "");`; + yield ` const ${names.queryParams} = new URLSearchParams(${names.ctx}.request.url!.split("?", 2)[1] ?? "");`; yield ""; } for (const qp of queryParams) { - yield* indent(emitQueryParamBinding(ctx, qp)); + yield* indent(emitQueryParamBinding(ctx, operation, names, qp)); } const bodyFields = new Map( @@ -156,17 +159,16 @@ function* emitRawServerOperation( const body = operation.parameters.body; if (body.contentTypes.length > 1) { - throw new UnimplementedError("dynamic request content type"); + reportDiagnostic(ctx.program, { + code: "dynamic-request-content-type", + target: operation.operation, + }); } const contentType = body.contentTypes[0] ?? DEFAULT_CONTENT_TYPE; const defaultBodyTypeName = operationNameCase.pascalCase + "RequestBody"; - if (body.bodyKind === "multipart") { - throw new UnimplementedError(`new form of multipart requests`); - } - const bodyNameCase = parseCase(body.property?.name ?? defaultBodyTypeName); const bodyTypeName = emitTypeReference( @@ -177,10 +179,22 @@ function* emitRawServerOperation( { altName: defaultBodyTypeName }, ); - bodyName = bodyNameCase.camelCase; + bodyName = ctx.gensym(bodyNameCase.camelCase); + + module.imports.push({ binder: ["parseHeaderValueParameters"], from: headerHelpers }); + + const contentTypeHeader = ctx.gensym("contentType"); + + yield ` const ${contentTypeHeader} = parseHeaderValueParameters(${names.ctx}.request.headers["content-type"] as string | undefined);`; + + yield ` if (${contentTypeHeader}?.value !== ${JSON.stringify(contentType)}) {`; + + yield ` return ${names.ctx}.errorHandlers.onInvalidRequest(`; + yield ` ${names.ctx},`; + yield ` ${JSON.stringify(operation.path)},`; + yield ` \`unexpected "Content-Type": '\${${contentTypeHeader}?.value}', expected '${JSON.stringify(contentType)}'\``; + yield ` );`; - yield ` if (!request.headers["content-type"]?.startsWith(${JSON.stringify(contentType)})) {`; - yield ` throw new Error(\`Invalid Request: expected content-type '${contentType}' but got '\${request.headers["content-type"]?.split(";", 2)[0]}'.\`)`; yield " }"; yield ""; @@ -190,52 +204,22 @@ function* emitRawServerOperation( requireSerialization(ctx, body.type as SerializableType, "application/json"); yield ` const ${bodyName} = await new Promise(function parse${bodyNameCase.pascalCase}(resolve, reject) {`; yield ` const chunks: Array = [];`; - yield ` request.on("data", function appendChunk(chunk) { chunks.push(chunk); });`; - yield ` request.on("end", function finalize() {`; + yield ` ${names.ctx}.request.on("data", function appendChunk(chunk) { chunks.push(chunk); });`; + yield ` ${names.ctx}.request.on("end", function finalize() {`; yield ` resolve(JSON.parse(Buffer.concat(chunks).toString()));`; yield ` });`; + yield ` ${names.ctx}.request.on("error", reject);`; yield ` }) as ${bodyTypeName};`; yield ""; break; } case "multipart/form-data": - yield `const ${bodyName} = await new Promise(function parse${bodyNameCase.pascalCase}MultipartRequest(resolve, reject) {`; - yield ` const boundary = request.headers["content-type"]?.split(";").find((s) => s.includes("boundary="))?.split("=", 2)[1];`; - yield ` if (!boundary) {`; - yield ` return reject("Invalid request: missing boundary in content-type.");`; - yield ` }`; - yield ""; - yield ` const chunks: Array = [];`; - yield ` request.on("data", function appendChunk(chunk) { chunks.push(chunk); });`; - yield ` request.on("end", function finalize() {`; - yield ` const text = Buffer.concat(chunks).toString();`; - yield ` const parts = text.split(boundary).slice(1, -1);`; - yield ` const fields: { [k: string]: any } = {};`; - yield ""; - yield ` for (const part of parts) {`; - yield ` const [headerText, body] = part.split("\\r\\n\\r\\n", 2);`; - yield " const headers = Object.fromEntries("; - yield ` headerText.split("\\r\\n").map((line) => line.split(": ", 2))`; - yield " ) as { [k: string]: string };"; - yield ` const name = headers["Content-Disposition"].split("name=\\"")[1].split("\\"")[0];`; - yield ` const contentType = headers["Content-Type"] ?? "text/plain";`; - yield ""; - yield ` switch (contentType) {`; - yield ` case "application/json":`; - yield ` fields[name] = JSON.parse(body);`; - yield ` break;`; - yield ` case "application/octet-stream":`; - yield ` fields[name] = Buffer.from(body, "utf-8");`; - yield ` break;`; - yield ` default:`; - yield ` fields[name] = body;`; - yield ` }`; - yield ` }`; - yield ""; - yield ` resolve(fields as ${bodyTypeName});`; - yield ` });`; - yield `}) as ${bodyTypeName};`; + if (body.bodyKind === "multipart") { + yield* indent(emitMultipart(ctx, module, operation, body, bodyName, bodyTypeName)); + } else { + yield* indent(emitMultipartLegacy(bodyName, bodyTypeName)); + } break; default: throw new UnimplementedError(`request deserialization for content-type: '${contentType}'`); @@ -253,8 +237,11 @@ function* emitRawServerOperation( let paramBaseExpression; const paramNameCase = parseCase(param.name); const isBodyField = bodyFields.has(param.name) && bodyFields.get(param.name) === param.type; + const isBodyExact = operation.parameters.body?.property === param; if (isBodyField) { paramBaseExpression = `${bodyName}.${paramNameCase.camelCase}`; + } else if (isBodyExact) { + paramBaseExpression = bodyName!; } else { const resolvedParameter = param.type.kind === "ModelProperty" ? param.type : param; @@ -283,18 +270,25 @@ function* emitRawServerOperation( ); } - yield ` const result = await operations.${operationNameCase.camelCase}(ctx, `; + yield ` const ${names.result} = await ${names.operations}.${operationNameCase.camelCase}(${names.ctx}, `; yield* indent(indent(paramLines)); // eslint-disable-next-line @typescript-eslint/no-unused-expressions yield ` );`, yield ""; - yield* indent(emitResultProcessing(ctx, op.returnType, module)); + yield* indent(emitResultProcessing(ctx, names, op.returnType, module)); yield "}"; yield ""; } +interface Names { + ctx: string; + result: string; + operations: string; + queryParams: string; +} + /** * Emit the result-processing code for an operation. * @@ -304,20 +298,25 @@ function* emitRawServerOperation( * @param t - The return type of the operation. * @param module - The module that the result processing code will be written to. */ -function* emitResultProcessing(ctx: HttpContext, t: Type, module: Module): Iterable { +function* emitResultProcessing( + ctx: HttpContext, + names: Names, + t: Type, + module: Module, +): Iterable { if (t.kind !== "Union") { // Single target type - yield* emitResultProcessingForType(ctx, t, module); + yield* emitResultProcessingForType(ctx, names, t, module); } else { const codeTree = differentiateUnion(ctx, t); yield* writeCodeTree(ctx, codeTree, { - subject: "result", + subject: names.result, referenceModelProperty(p) { - return "result." + parseCase(p.name).camelCase; + return names.result + "." + parseCase(p.name).camelCase; }, // We mapped the output directly in the code tree input, so we can just return it. - renderResult: (t) => emitResultProcessingForType(ctx, t, module), + renderResult: (t) => emitResultProcessingForType(ctx, names, t, module), }); } } @@ -331,6 +330,7 @@ function* emitResultProcessing(ctx: HttpContext, t: Type, module: Module): Itera */ function* emitResultProcessingForType( ctx: HttpContext, + names: Names, target: Type, module: Module, ): Iterable { @@ -343,8 +343,8 @@ function* emitResultProcessingForType( for (const property of target.properties.values()) { if (isHeader(ctx.program, property)) { const headerName = getHeaderFieldName(ctx.program, property); - yield `response.setHeader(${JSON.stringify(headerName.toLowerCase())}, result.${parseCase(property.name).camelCase});`; - if (!body) yield `delete (result as any).${parseCase(property.name).camelCase};`; + yield `${names.ctx}.response.setHeader(${JSON.stringify(headerName.toLowerCase())}, ${names.result}.${parseCase(property.name).camelCase});`; + if (!body) yield `delete (${names.result} as any).${parseCase(property.name).camelCase};`; } else if (isStatusCode(ctx.program, property)) { if (isUnspeakable(property.name)) { if (!isValueLiteralType(property.type)) { @@ -360,10 +360,10 @@ function* emitResultProcessingForType( compilerAssert(property.type.kind === "Number", "Status code must be a number."); - yield `response.statusCode = ${property.type.valueAsString};`; + yield `${names.ctx}.response.statusCode = ${property.type.valueAsString};`; } else { - yield `response.statusCode = result.${parseCase(property.name).camelCase};`; - if (!body) yield `delete (result as any).${parseCase(property.name).camelCase};`; + yield `${names.ctx}.response.statusCode = ${names.result}.${parseCase(property.name).camelCase};`; + if (!body) yield `delete (${names.result} as any).${parseCase(property.name).camelCase};`; } } } @@ -382,13 +382,13 @@ function* emitResultProcessingForType( const typeReference = emitTypeReference(ctx, body.type, body, module, { requireDeclaration: true, }); - yield `response.end(JSON.stringify(${typeReference}.toJsonObject(result.${bodyCase.camelCase})))`; + yield `${names.ctx}.response.end(JSON.stringify(${typeReference}.toJsonObject(${names.result}.${bodyCase.camelCase})))`; } else { - yield `response.end(JSON.stringify(result.${bodyCase.camelCase}));`; + yield `${names.ctx}.response.end(JSON.stringify(${names.result}.${bodyCase.camelCase}));`; } } else { if (allMetadataIsRemoved) { - yield `response.end();`; + yield `${names.ctx}.response.end();`; } else { const serializationRequired = isSerializationRequired(ctx, target, "application/json"); requireSerialization(ctx, target, "application/json"); @@ -396,9 +396,9 @@ function* emitResultProcessingForType( const typeReference = emitTypeReference(ctx, target, target, module, { requireDeclaration: true, }); - yield `response.end(JSON.stringify(${typeReference}.toJsonObject(result as ${typeReference})));`; + yield `${names.ctx}.response.end(JSON.stringify(${typeReference}.toJsonObject(${names.result} as ${typeReference})));`; } else { - yield `response.end(JSON.stringify(result));`; + yield `${names.ctx}.response.end(JSON.stringify(${names.result}));`; } } } @@ -414,6 +414,8 @@ function* emitResultProcessingForType( */ function* emitHeaderParamBinding( ctx: HttpContext, + operation: HttpOperation, + names: Names, parameter: Extract, ): Iterable { const nameCase = parseCase(parameter.param.name); @@ -424,11 +426,12 @@ function* emitHeaderParamBinding( const assertion = canBeArrayType ? "" : " as string | undefined"; - yield `const ${nameCase.camelCase} = request.headers[${JSON.stringify(parameter.name)}]${assertion};`; + yield `const ${nameCase.camelCase} = ${names.ctx}.request.headers[${JSON.stringify(parameter.name)}]${assertion};`; if (!parameter.param.optional) { yield `if (${nameCase.camelCase} === undefined) {`; // prettier-ignore + yield ` return ${names.ctx}.errorHandlers.onInvalidRequest(${names.ctx}, ${JSON.stringify(operation.path)}, "missing required header '${parameter.name}'");`; yield ` throw new Error("Invalid request: missing required header '${parameter.name}'.");`; yield "}"; yield ""; @@ -445,17 +448,18 @@ function* emitHeaderParamBinding( */ function* emitQueryParamBinding( ctx: HttpContext, + operation: HttpOperation, + names: Names, parameter: Extract, ): Iterable { const nameCase = parseCase(parameter.param.name); // UrlSearchParams annoyingly returns null for missing parameters instead of undefined. - yield `const ${nameCase.camelCase} = __query_params.get(${JSON.stringify(parameter.name)}) ?? undefined;`; + yield `const ${nameCase.camelCase} = ${names}.get(${JSON.stringify(parameter.name)}) ?? undefined;`; if (!parameter.param.optional) { yield `if (!${nameCase.camelCase}) {`; - // prettier-ignore - yield ` throw new Error("Invalid request: missing required query parameter '${parameter.name}'.");`; + yield ` ${names.ctx}.errorHandlers.onInvalidRequest(${names.ctx}, ${JSON.stringify(operation.path)}, "missing required query parameter '${parameter.name}');`; yield "}"; yield ""; } diff --git a/packages/http-server-javascript/src/http/server/multipart.ts b/packages/http-server-javascript/src/http/server/multipart.ts new file mode 100644 index 00000000000..6b973f2e513 --- /dev/null +++ b/packages/http-server-javascript/src/http/server/multipart.ts @@ -0,0 +1,247 @@ +import { HttpOperation, HttpOperationMultipartBody, isHttpFile } from "@typespec/http"; +import { Module } from "../../ctx.js"; +import { HttpContext } from "../index.js"; + +import { module as headerHelpers } from "../../../generated-defs/helpers/header.js"; +import { module as multipartHelpers } from "../../../generated-defs/helpers/multipart.js"; +import { parseCase } from "../../util/case.js"; +import { UnimplementedError } from "../../util/error.js"; + +/** + * Parse a multipart request body according to the given body spec. + * + * @param ctx - The emitter context. + * @param module - The module that this parser is written into. + * @param operation - The HTTP operation this body is being parsed for. + * @param body - The multipart body spec + * @param bodyName - The name of the variable to store the parsed body in. + * @param bodyTypeName - The name of the type of the parsed body. + */ +export function* emitMultipart( + ctx: HttpContext, + module: Module, + operation: HttpOperation, + body: HttpOperationMultipartBody, + bodyName: string, + bodyTypeName: string, +): Iterable { + module.imports.push( + { binder: ["parseHeaderValueParameters"], from: headerHelpers }, + { binder: ["createMultipartReadable"], from: multipartHelpers }, + ); + + yield `const ${bodyName} = await new Promise<${bodyTypeName}>(`; + yield `// eslint-disable-next-line no-async-promise-executor`; + yield `async function parse${bodyTypeName}MultipartRequest(resolve, reject) {`; + + // Wrap this whole thing in a try/catch because the executor is async. If anything in here throws, we want to reject the promise instead of + // just letting the executor die and the promise never settle. + yield ` try {`; + + const stream = ctx.gensym("stream"); + + yield ` const ${stream} = createMultipartReadable(request);`; + yield ""; + + const contentDisposition = ctx.gensym("contentDisposition"); + const contentType = ctx.gensym("contentType"); + const name = ctx.gensym("name"); + const fields = ctx.gensym("fields"); + + yield ` const ${fields}: { [k: string]: any } = {};`; + yield ""; + + const partsWithMulti = body.parts.filter((part) => part.name && part.multi); + const anonymousParts = body.parts.filter((part) => !part.name); + const anonymousPartsAreMulti = anonymousParts.some((part) => part.multi); + + if (anonymousParts.length > 0) { + throw new UnimplementedError("Anonymous parts are not yet supported in multipart parsing."); + } + + let hadMulti = false; + + for (const partWithMulti of partsWithMulti) { + if (!partWithMulti.optional) { + hadMulti = true; + const name = partWithMulti.name!; + + const propName = parseCase(name).camelCase; + + yield ` ${fields}.${propName} = [];`; + } + } + + if (anonymousPartsAreMulti) { + hadMulti = true; + yield ` const ${fields}.__anonymous = [];`; + } + + if (hadMulti) yield ""; + + const partName = ctx.gensym("part"); + + yield ` for await (const ${partName} of ${stream}) {`; + yield ` const ${contentDisposition} = parseHeaderValueParameters(${partName}.headers["content-disposition"]);`; + yield ` if (!${contentDisposition}) {`; + yield ` return reject("Invalid request: missing content-disposition in part.");`; + yield ` }`; + yield ""; + yield ` const ${contentType} = parseHeaderValueParameters(${partName}.headers["content-type"]);`; + yield ""; + yield ` const ${name} = ${contentDisposition}.params.name ?? "";`; + yield ""; + yield ` switch (${name}) {`; + + for (const namedPart of body.parts.filter((part) => part.name)) { + // TODO: this is wrong. The name of the part is not necessarily the name of the property in the body. + // The HTTP library does not provide the property that maps to this part if it's explicitly named. + const propName = parseCase(namedPart.name!).camelCase; + + let value = ctx.gensym("value"); + + yield ` case ${JSON.stringify(namedPart.name)}: {`; + // HTTP API is doing too much work for us. I need to know whether I'm looking at an HTTP file, and the only way to do that is to + // look at the model that the body is a property of. This is more than a bit of a hack, but it will work for now. + if ( + namedPart.body.contentTypeProperty?.model && + isHttpFile(ctx.program, namedPart.body.contentTypeProperty.model) + ) { + // We have an http file, so we will buffer the body and then optionally get the filename and content type. + // TODO: support models that inherit from File and have other optional metadata. The Http.File structure + // doesn't make this easy to do, since it doesn't describe where the fields of the file come from in the + // multipart request. However, we could recognize models that extend File and handle the special fields + // of Http.File specially. + // TODO: find a way to avoid buffering the entire file in memory. I have to do this to return an object that + // has the keys described in the TypeSpec model and because the underlying multipart stream has to be + // drained sequentially. Server authors could stall the stream by trying to read part bodies out of order if + // I represented the file contents as a stream. We will need some way to identify the whole multipart + // envelope and represent it as a stream of named parts. The backend for multipart streaming supports this, + // and it's how we receive the part data in this handler, but we don't have a way to represent it to the + // implementor yet. + + yield ` const __chunks = [];`; + yield ""; + yield ` for await (const __chunk of ${partName}.body) {`; + yield ` __chunks.push(__chunk);`; + yield ` }`; + yield ""; + + yield ` const ${value}: { filename?: string; contentType?: string; contents: Buffer; } = { contents: Buffer.concat(__chunks) };`; + yield ""; + + yield ` if (${contentType}) {`; + yield ` ${value}.contentType = ${contentType}.verbatim;`; + yield ` }`; + yield ""; + + yield ` const __filename = ${contentDisposition}.params.filename;`; + yield ` if (__filename) {`; + yield ` ${value}.filename = __filename;`; + yield ` }`; + } else { + // Not a file. We just use the given content-type to determine how to parse the body. + + yield ` if (${contentType}?.value && ${contentType}.value !== "application/json") {`; + yield ` throw new Error("Unsupported content-type for part: " + ${contentType}.value);`; + yield ` }`; + yield ""; + + if (namedPart.headers.length > 0) { + // TODO: support reconstruction of mixed objects with headers and bodies. + throw new UnimplementedError( + "Named parts with headers are not yet supported in multipart parsing.", + ); + } + + yield ` let __chunks = Buffer.alloc(0);`; + yield ""; + yield ` for await (const __chunk of ${partName}.body) {`; + yield ` __chunks.push(__chunk);`; + yield ` }`; + yield ""; + value = 'JSON.parse(Buffer.concat(__chunks).toString("utf-8"));'; + } + if (namedPart.multi) { + if (namedPart.optional) { + yield ` (${fields}.${propName} ??= []).push(${value});`; + } else { + yield ` ${fields}.${propName}.push(${value});`; + } + } else { + yield ` ${fields}.${propName} = ${value};`; + } + yield ` break;`; + yield ` }`; + } + + if (anonymousParts.length > 0) { + yield ` "": {`; + if (anonymousPartsAreMulti) { + yield ` ${fields}.__anonymous.push({`; + yield ` headers: ${partName}.headers,`; + yield ` body: ${partName}.body,`; + yield ` });`; + yield ` break;`; + } else { + yield ` ${fields}.__anonymous = {}`; + yield ` break;`; + } + yield ` }`; + } + + yield ` default: {`; + yield ` reject("Invalid request: unknown part name.");`; + yield ` return;`; + yield ` }`; + yield ` }`; + yield ` }`; + yield ""; + + yield ` resolve(${fields} as ${bodyTypeName});`; + + yield ` } catch (err) { reject(err); }`; + + yield "});"; +} + +// This function is old and broken. I'm not likely to fix it unless we decide to continue supporting legacy multipart +// parsing after 1.0. +export function* emitMultipartLegacy(bodyName: string, bodyTypeName: string): Iterable { + yield `const ${bodyName} = await new Promise(function parse${bodyTypeName}MultipartRequest(resolve, reject) {`; + yield ` const boundary = request.headers["content-type"]?.split(";").find((s) => s.includes("boundary="))?.split("=", 2)[1];`; + yield ` if (!boundary) {`; + yield ` return reject("Invalid request: missing boundary in content-type.");`; + yield ` }`; + yield ""; + yield ` const chunks: Array = [];`; + yield ` request.on("data", function appendChunk(chunk) { chunks.push(chunk); });`; + yield ` request.on("end", function finalize() {`; + yield ` const text = Buffer.concat(chunks).toString();`; + yield ` const parts = text.split(boundary).slice(1, -1);`; + yield ` const fields: { [k: string]: any } = {};`; + yield ""; + yield ` for (const part of parts) {`; + yield ` const [headerText, body] = part.split("\\r\\n\\r\\n", 2);`; + yield " const headers = Object.fromEntries("; + yield ` headerText.split("\\r\\n").map((line) => line.split(": ", 2))`; + yield " ) as { [k: string]: string };"; + yield ` const name = headers["Content-Disposition"].split("name=\\"")[1].split("\\"")[0];`; + yield ` const contentType = headers["Content-Type"] ?? "text/plain";`; + yield ""; + yield ` switch (contentType) {`; + yield ` case "application/json":`; + yield ` fields[name] = JSON.parse(body);`; + yield ` break;`; + yield ` case "application/octet-stream":`; + yield ` fields[name] = Buffer.from(body, "utf-8");`; + yield ` break;`; + yield ` default:`; + yield ` fields[name] = body;`; + yield ` }`; + yield ` }`; + yield ""; + yield ` resolve(fields as ${bodyTypeName});`; + yield ` });`; + yield `}) as ${bodyTypeName};`; +} diff --git a/packages/http-server-javascript/src/http/server/router.ts b/packages/http-server-javascript/src/http/server/router.ts index 718f98f2e51..1dadb729075 100644 --- a/packages/http-server-javascript/src/http/server/router.ts +++ b/packages/http-server-javascript/src/http/server/router.ts @@ -137,28 +137,38 @@ function* emitRouterDefinition( yield ` }> = {}`; yield `): ${routerName} {`; + const [onRequestNotFound, onInvalidRequest, onInternalError] = [ + "onRequestNotFound", + "onInvalidRequest", + "onInternalError", + ].map(ctx.gensym); + // Router error case handlers - yield ` const onRouteNotFound = options.onRequestNotFound ?? ((request, response) => {`; - yield ` response.statusCode = 404;`; - yield ` response.setHeader("Content-Type", "text/plain");`; - yield ` response.end("Not Found");`; + yield ` const ${onRequestNotFound} = options.onRequestNotFound ?? ((ctx) => {`; + yield ` ctx.response.statusCode = 404;`; + yield ` ctx.response.setHeader("Content-Type", "text/plain");`; + yield ` ctx.response.end("Not Found");`; yield ` });`; yield ""; - yield ` const onInvalidRequest = options.onInvalidRequest ?? ((request, response, route, error) => {`; - yield ` response.statusCode = 400;`; - yield ` response.setHeader("Content-Type", "application/json");`; - yield ` response.end(JSON.stringify({ error }));`; + yield ` const ${onInvalidRequest} = options.onInvalidRequest ?? ((ctx, route, error) => {`; + yield ` ctx.response.statusCode = 400;`; + yield ` ctx.response.setHeader("Content-Type", "application/json");`; + yield ` ctx.response.end(JSON.stringify({ error }));`; yield ` });`; yield ""; - yield ` const onInternalError = options.onInternalError ?? ((error, request, response) => {`; - yield ` response.statusCode = 500;`; - yield ` response.setHeader("Content-Type", "text/plain");`; - yield ` response.end("Internal server error.");`; + yield ` const ${onInternalError} = options.onInternalError ?? ((ctx, error) => {`; + yield ` ctx.response.statusCode = 500;`; + yield ` ctx.response.setHeader("Content-Type", "text/plain");`; + yield ` ctx.response.end("Internal server error.");`; yield ` });`; yield ""; - yield ` const routePolicies = options.routePolicies ?? {};`; + + const routePolicies = ctx.gensym("routePolicies"); + const routeHandlers = ctx.gensym("routeHandlers"); + + yield ` const ${routePolicies} = options.routePolicies ?? {};`; yield ""; - yield ` const routeHandlers = {`; + yield ` const ${routeHandlers} = {`; // Policy chains for each operation for (const operation of service.operations) { @@ -167,7 +177,7 @@ function* emitRouterDefinition( yield ` ${containerName.snakeCase}_${operationName.snakeCase}: createPolicyChainForRoute(`; yield ` "${containerName.camelCase + operationName.pascalCase + "Dispatch"}",`; - yield ` routePolicies,`; + yield ` ${routePolicies},`; yield ` "${containerName.camelCase}",`; yield ` "${operationName.camelCase}",`; yield ` serverRaw.${containerName.snakeCase}_${operationName.snakeCase},`; @@ -178,23 +188,45 @@ function* emitRouterDefinition( yield ""; // Core routing function definition - yield ` const dispatch = createPolicyChain("${routerName}Dispatch", options.policies ?? [], async function(ctx, request, response, onRouteNotFound) {`; + yield ` const dispatch = createPolicyChain("${routerName}Dispatch", options.policies ?? [], async function(ctx, request, response) {`; yield ` const url = new URL(request.url!, \`http://\${request.headers.host}\`);`; yield ` let path = url.pathname;`; yield ""; - yield* indent(indent(emitRouteHandler(ctx, routeTree, backends, module))); + yield* indent(indent(emitRouteHandler(ctx, routeHandlers, routeTree, backends, module))); yield ""; - yield ` return onRouteNotFound(request, response);`; + yield ` return ${onRequestNotFound}(ctx);`; yield ` });`; yield ""; + + const errorHandlers = ctx.gensym("errorHandlers"); + + yield ` const ${errorHandlers} = {`; + yield ` onRequestNotFound: ${onRequestNotFound},`; + yield ` onInvalidRequest: ${onInvalidRequest},`; + yield ` onInternalError: ${onInternalError},`; + yield ` };`; + yield ` return {`; - yield ` dispatch(request, response) { return dispatch({ request, response }, request, response, onRouteNotFound).catch((e) => onInternalError(e, request, response)); },`; + yield ` dispatch(request, response) {`; + yield ` const ctx = { request, response, errorHandlers: ${errorHandlers} };`; + yield ` return dispatch(ctx, request, response).catch((e) => ${onInternalError}(ctx, e));`; + yield ` },`; if (ctx.options.express) { - yield ` expressMiddleware: function (request, response, next) { void dispatch({ request, response }, request, response, function () { next(); }).catch((e) => onInternalError(e, request, response)); },`; + yield ` expressMiddleware: function (request, response, next) {`; + yield ` const ctx = { request, response, errorHandlers: ${errorHandlers} };`; + yield ` void dispatch(`; + yield ` { request, response, errorHandlers: {`; + yield ` ...${errorHandlers},`; + yield ` onRequestNotFound: function () { next() }`; + yield ` }},`; + yield ` request,`; + yield ` response`; + yield ` ).catch((e) => ${onInternalError}(ctx, e));`; + yield ` },`; } yield " }"; @@ -211,25 +243,28 @@ function* emitRouterDefinition( */ function* emitRouteHandler( ctx: HttpContext, + routeHandlers: string, routeTree: RouteTree, backends: Map, module: Module, ): Iterable { const mustTerminate = routeTree.edges.length === 0 && !routeTree.bind; + const onRouteNotFound = "ctx.errorHandlers.onRequestNotFound"; + yield `if (path.length === 0) {`; if (routeTree.operations.size > 0) { - yield* indent(emitRouteOperationDispatch(ctx, routeTree.operations, backends)); + yield* indent(emitRouteOperationDispatch(ctx, routeHandlers, routeTree.operations, backends)); } else { // Not found - yield ` return onRouteNotFound(request, response);`; + yield ` return ${onRouteNotFound}(ctx);`; } yield `}`; if (mustTerminate) { // Not found yield "else {"; - yield ` return onRouteNotFound(request, response);`; + yield ` return ${onRouteNotFound}(ctx);`; yield `}`; return; } @@ -238,7 +273,7 @@ function* emitRouteHandler( const edgePattern = edge.length === 1 ? `'${edge}'` : JSON.stringify(edge); yield `else if (path.startsWith(${edgePattern})) {`; yield ` path = path.slice(${edge.length});`; - yield* indent(emitRouteHandler(ctx, nextTree, backends, module)); + yield* indent(emitRouteHandler(ctx, routeHandlers, nextTree, backends, module)); yield "}"; } @@ -258,7 +293,7 @@ function* emitRouteHandler( yield ` const ${parseCase(p).camelCase} = param;`; } } - yield* indent(emitRouteHandler(ctx, nextTree, backends, module)); + yield* indent(emitRouteHandler(ctx, routeHandlers, nextTree, backends, module)); yield `}`; } @@ -273,6 +308,7 @@ function* emitRouteHandler( */ function* emitRouteOperationDispatch( ctx: HttpContext, + routeHandlers: string, operations: Map, backends: Map, ): Iterable { @@ -293,19 +329,21 @@ function* emitRouteOperationDispatch( : ""; yield ` case ${JSON.stringify(verb.toUpperCase())}:`; - yield ` return routeHandlers.${operationName}(ctx, request, response, ${backendMemberName}${parameters});`; + yield ` return ${routeHandlers}.${operationName}(ctx, ${backendMemberName}${parameters});`; } else { // Shared route const route = getHttpOperation(ctx.program, operationList[0].operation)[0].path; yield ` case ${JSON.stringify(verb.toUpperCase())}:`; yield* indent( - indent(emitRouteOperationDispatchMultiple(ctx, operationList, route, backends)), + indent( + emitRouteOperationDispatchMultiple(ctx, routeHandlers, operationList, route, backends), + ), ); } } yield ` default:`; - yield ` return onRouteNotFound(request, response);`; + yield ` return ctx.errorHandlers.onRequestNotFound(ctx);`; yield "}"; } @@ -319,6 +357,7 @@ function* emitRouteOperationDispatch( */ function* emitRouteOperationDispatchMultiple( ctx: HttpContext, + routeHandlers: string, operations: RouteOperation[], route: string, backends: Map, @@ -350,8 +389,7 @@ function* emitRouteOperationDispatchMultiple( contentTypeMap.set(operation, operationContentType.value); } - yield `const contentType = request.headers["content-type"];`; - yield `switch (contentType) {`; + yield `switch (request.headers["content-type"]) {`; for (const [operation, contentType] of contentTypeMap.entries()) { const [backend] = backends.get(operation.container)!; @@ -367,11 +405,11 @@ function* emitRouteOperationDispatchMultiple( : ""; yield ` case ${JSON.stringify(contentType)}:`; - yield ` return routeHandlers.${operationName}(ctx, request, response, ${backendMemberName}${parameters});`; + yield ` return ${routeHandlers}.${operationName}(ctx, ${backendMemberName}${parameters});`; } yield ` default:`; - yield ` return onInvalidRequest(request, response, ${JSON.stringify(route)}, \`No operation in route '${route}' matched content-type "\${contentType}"\`);`; + yield ` return ctx.errorHandlers.onInvalidRequest(ctx, ${JSON.stringify(route)}, \`No operation in route '${route}' matched content-type "\${contentType}"\`);`; yield "}"; } diff --git a/packages/http-server-javascript/src/index.ts b/packages/http-server-javascript/src/index.ts index fd3cecadd6b..b0a66e7e22b 100644 --- a/packages/http-server-javascript/src/index.ts +++ b/packages/http-server-javascript/src/index.ts @@ -3,7 +3,7 @@ import { EmitContext, NoTarget, listServices } from "@typespec/compiler"; import { visitAllTypes } from "./common/namespace.js"; -import { JsContext, Module, createModule, createPathCursor } from "./ctx.js"; +import { JsContext, Module, createModule, createPathCursor, gensym } from "./ctx.js"; import { JsEmitterOptions, reportDiagnostic } from "./lib.js"; import { parseCase } from "./util/case.js"; import { UnimplementedError } from "./util/error.js"; @@ -82,6 +82,10 @@ export async function $onEmit(context: EmitContext) { globalNamespaceModule: allModule, serializations: createOnceQueue(), + + gensym: (name) => { + return gensym(jsCtx, name); + }, }; await emitHttp(jsCtx); diff --git a/packages/http-server-javascript/src/lib.ts b/packages/http-server-javascript/src/lib.ts index 26c853e6572..df516034729 100644 --- a/packages/http-server-javascript/src/lib.ts +++ b/packages/http-server-javascript/src/lib.ts @@ -107,6 +107,12 @@ export const $lib = createTypeSpecLibrary({ default: paramMessage`Name ${"name"} conflicts with a prior declaration and must be unique.`, }, }, + "dynamic-request-content-type": { + severity: "error", + messages: { + default: "Operation has multiple possible content-type values and cannot be emitted.", + }, + }, }, }); diff --git a/packages/http-server-javascript/src/util/name.ts b/packages/http-server-javascript/src/util/name.ts index f1077a92bd7..e259cdf95b6 100644 --- a/packages/http-server-javascript/src/util/name.ts +++ b/packages/http-server-javascript/src/util/name.ts @@ -14,7 +14,9 @@ export type NamespacedType = Extract"; if (type.namespace) { - return getFullyQualifiedNamespacePath(type.namespace).join(".") + "." + name; + const nsPath = getFullyQualifiedNamespacePath(type.namespace); + + return (nsPath[0] === "" ? nsPath.slice(1) : nsPath).join(".") + "." + name; } else { return name; } diff --git a/packages/http-server-javascript/test/header.test.ts b/packages/http-server-javascript/test/header.test.ts new file mode 100644 index 00000000000..4227752e16f --- /dev/null +++ b/packages/http-server-javascript/test/header.test.ts @@ -0,0 +1,26 @@ +import { assert, describe, it } from "vitest"; +import { parseHeaderValueParameters } from "../src/helpers/header.js"; + +describe("headers", () => { + it("parses header values with parameters", () => { + const { value, params } = parseHeaderValueParameters("text/html; charset=utf-8"); + + assert.equal(value, "text/html"); + assert.equal(params.charset, "utf-8"); + }); + + it("parses a header value with a quoted parameter", () => { + const { value, params } = parseHeaderValueParameters('text/html; charset="utf-8"'); + + assert.equal(value, "text/html"); + assert.equal(params.charset, "utf-8"); + }); + + it("parses a header value with multiple parameters", () => { + const { value, params } = parseHeaderValueParameters('text/html; charset="utf-8"; foo=bar'); + + assert.equal(value, "text/html"); + assert.equal(params.charset, "utf-8"); + assert.equal(params.foo, "bar"); + }); +}); diff --git a/packages/http-server-javascript/test/multipart.test.ts b/packages/http-server-javascript/test/multipart.test.ts new file mode 100644 index 00000000000..d62a7acd492 --- /dev/null +++ b/packages/http-server-javascript/test/multipart.test.ts @@ -0,0 +1,169 @@ +import { EventEmitter } from "stream"; +import { assert, describe, it } from "vitest"; +import { createMultipartReadable } from "../src/helpers/multipart.js"; + +import type * as http from "node:http"; + +interface StringChunkOptions { + sizeConstraint: [number, number]; + timeConstraintMs: [number, number]; +} + +function chunkString(s: string, options: StringChunkOptions): EventEmitter { + const [min, max] = options.sizeConstraint; + const [minTime, maxTime] = options.timeConstraintMs; + const emitter = new EventEmitter(); + let i = 0; + + function emitChunk() { + const chunkSize = Math.floor(Math.random() * (max - min + 1) + min); + emitter.emit("data", Buffer.from(s.slice(i, i + chunkSize))); + i += chunkSize; + } + + setTimeout( + function tick() { + emitChunk(); + + if (i < s.length) { + setTimeout(tick, Math.floor(Math.random() * (maxTime - minTime + 1) + minTime)); + } else { + emitter.emit("end"); + } + }, + Math.floor(Math.random() * (maxTime - minTime + 1) + minTime), + ); + + return emitter; +} + +const exampleMultipart = [ + "This is the preamble text. It should be ignored.", + "--boundary", + 'Content-Disposition: form-data; name="field1"', + "Content-Type: application/json", + "", + '"value1"', + "--boundary", + 'Content-Disposition: form-data; name="field2"', + "", + "value2", + "--boundary--", +].join("\r\n"); + +function createMultipartRequestLike( + text: string, + boundary: string = "boundary", +): http.IncomingMessage { + return Object.assign( + chunkString(text, { sizeConstraint: [40, 90], timeConstraintMs: [20, 30] }), + { + headers: { "content-type": `multipart/form-data; boundary=${boundary}` }, + }, + ) as any; +} + +describe("multipart", () => { + it("correctly chunks multipart data", async () => { + const request = createMultipartRequestLike(exampleMultipart); + + const stream = createMultipartReadable(request); + + const parts: Array<{ headers: { [k: string]: string | undefined }; body: string }> = []; + + for await (const part of stream) { + parts.push({ + headers: part.headers, + body: await (async () => { + const chunks = []; + for await (const chunk of part.body) { + chunks.push(chunk); + } + return Buffer.concat(chunks).toString(); + })(), + }); + } + + assert.deepStrictEqual(parts, [ + { + headers: { + "content-disposition": 'form-data; name="field1"', + "content-type": "application/json", + }, + body: '"value1"', + }, + { + headers: { "content-disposition": 'form-data; name="field2"' }, + body: "value2", + }, + ]); + }); + + it("detects missing boundary", () => { + assert.throws(() => { + createMultipartReadable({ headers: {} } as any); + }, "missing boundary"); + + assert.throws(() => { + createMultipartReadable({ + headers: { "content-type": "multipart/form-data" }, + } as any); + }, "missing boundary"); + }); + + it("detects unexpected termination", async () => { + const request = createMultipartRequestLike( + [ + "--boundary", + 'Content-Disposition: form-data; name="field1"', + "Content-Type: application/json", + "", + '"value1"', + "--boundary asdf asdf", + ].join("\r\n"), + ); + + const stream = createMultipartReadable(request); + + try { + for await (const part of stream) { + for await (const _ of part.body) { + // Do nothing + } + } + assert.fail(); + } catch (e) { + assert.equal((e as Error).message, "Unexpected characters after final boundary."); + } + }); + + it("detects invalid preamble text", async () => { + const request = createMultipartRequestLike( + [ + "This is the preamble text. It should be ignored.--boundary", + 'Content-Disposition: form-data; name="field1"', + "Content-Type: application/json", + "", + '"value1"', + "--boundary", + 'Content-Disposition: form-data; name="field2"', + "", + "value2", + "--boundary--", + ].join("\r\n"), + ); + + const stream = createMultipartReadable(request); + + try { + for await (const part of stream) { + for await (const _ of part.body) { + // Do nothing + } + } + assert.fail(); + } catch (e) { + assert.equal((e as Error).message, "Invalid preamble in multipart body."); + } + }); +}); diff --git a/packages/http-server-javascript/vitest.config.ts b/packages/http-server-javascript/vitest.config.ts new file mode 100644 index 00000000000..15eeaceb856 --- /dev/null +++ b/packages/http-server-javascript/vitest.config.ts @@ -0,0 +1,4 @@ +import { defineConfig, mergeConfig } from "vitest/config"; +import { defaultTypeSpecVitestConfig } from "../../vitest.workspace.js"; + +export default mergeConfig(defaultTypeSpecVitestConfig, defineConfig({})); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1ec21bf0d36..a88ca5b0c39 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -630,12 +630,21 @@ importers: '@typespec/http': specifier: workspace:~ version: link:../http + '@vitest/coverage-v8': + specifier: ^2.1.5 + version: 2.1.5(vitest@2.1.5(@types/node@22.7.9)(@vitest/ui@2.1.5)(happy-dom@15.11.6)(jsdom@19.0.0)) + '@vitest/ui': + specifier: ^2.1.2 + version: 2.1.5(vitest@2.1.5) tsx: specifier: ^4.19.2 version: 4.19.2 typescript: specifier: ~5.6.3 version: 5.6.3 + vitest: + specifier: ^2.1.5 + version: 2.1.5(@types/node@22.7.9)(@vitest/ui@2.1.5)(happy-dom@15.11.6)(jsdom@19.0.0) packages/http-specs: dependencies: From 77df8c462ea29e48a66de001b886e8e3fded9803 Mon Sep 17 00:00:00 2001 From: Will Temple Date: Mon, 6 Jan 2025 16:29:54 -0500 Subject: [PATCH 2/3] Improved multipart handling, body parsing logic, etc. --- .../generated-defs/helpers/multipart.ts | 24 +++++----- .../src/common/scalar.ts | 24 ++++++++++ .../src/common/serialization/json.ts | 38 +++++++++++++--- .../src/helpers/multipart.ts | 8 ++-- .../src/http/server/index.ts | 44 +++++++++++++++---- .../src/http/server/multipart.ts | 39 +++++++++++++--- .../src/http/server/router.ts | 21 +++++++-- 7 files changed, 159 insertions(+), 39 deletions(-) diff --git a/packages/http-server-javascript/generated-defs/helpers/multipart.ts b/packages/http-server-javascript/generated-defs/helpers/multipart.ts index edb37e31409..8f058c913e6 100644 --- a/packages/http-server-javascript/generated-defs/helpers/multipart.ts +++ b/packages/http-server-javascript/generated-defs/helpers/multipart.ts @@ -35,6 +35,8 @@ const lines = [ " let boundarySplit = Buffer.from(`--${boundary}`);", " let initialized = false;", "", + " // We need to keep at least the length of the boundary split plus room for CRLFCRLF in the buffer to detect the boundaries.", + " // We subtract one from this length because if the whole thing were in the buffer, we would detect it and move past it.", " const bufferKeepLength = boundarySplit.length + BUF_CRLFCRLF.length - 1;", " let _readableController: ReadableStreamDefaultController> = null as any;", "", @@ -50,12 +52,12 @@ const lines = [ " write: async (chunk) => {", " buffer = Buffer.concat([buffer, chunk]);", "", - " const index = buffer.indexOf(boundarySplit);", + " let index: number;", "", - " if (index !== -1) {", + " while ((index = buffer.indexOf(boundarySplit)) !== -1) {", " // We found a boundary, emit everything before it and initialize a new stream for the next part.", "", - " // We are initialized if we have consumed at least one chunk.", + " // We are initialized if we have found the boundary at least once.", " //", " // Cases", " // 1. If the index is zero and we aren't initialized, there was no preamble.", @@ -84,23 +86,25 @@ const lines = [ " // subcontroller yet.", " subController?.close();", " subController = null;", + "", + " if (!initialized) {", + " initialized = true;", + " boundarySplit = Buffer.from(`\\r\\n${boundarySplit}`);", + " }", " }", "", " if (buffer.length > bufferKeepLength) {", " await enqueueSub(buffer.subarray(0, -bufferKeepLength));", " buffer = buffer.subarray(-bufferKeepLength);", " }", - "", - " if (!initialized) {", - " initialized = true;", - " boundarySplit = Buffer.from(`\\r\\n${boundarySplit}`);", - " }", " },", " close() {", - " if (buffer.toString(\"utf-8\") !== \"--\") {", - " readableController.error(new Error(\"Multipart body terminated unexpectedly.\"));", + " if (!/--(\\r\\n)?/.test(buffer.toString(\"utf-8\"))) {", + " readableController.error(new Error(\"Unexpected characters after final boundary.\"));", " }", "", + " subController?.close();", + "", " readableController.close();", " },", " });", diff --git a/packages/http-server-javascript/src/common/scalar.ts b/packages/http-server-javascript/src/common/scalar.ts index 4948be6d0be..868b178e6c7 100644 --- a/packages/http-server-javascript/src/common/scalar.ts +++ b/packages/http-server-javascript/src/common/scalar.ts @@ -52,11 +52,35 @@ export function parseTemplateForScalar(ctx: JsContext, scalar: Scalar): string { return "Number({})"; case "bigint": return "BigInt({})"; + case "Uint8Array": + return "Buffer.from({}, 'base64')"; default: throw new UnimplementedError(`parse template for scalar '${jsScalar}'`); } } +/** + * Get the string encoding template for a given scalar. + * @param ctx + * @param scalar + */ +export function encodeTemplateForScalar(ctx: JsContext, scalar: Scalar): string { + const jsScalar = getJsScalar(ctx.program, scalar, scalar); + + switch (jsScalar) { + case "string": + return "{}"; + case "number": + return "String({})"; + case "bigint": + return "String({})"; + case "Uint8Array": + return "{}.toString('base64')"; + default: + throw new UnimplementedError(`encode template for scalar '${jsScalar}'`); + } +} + const __JS_SCALARS_MAP = new Map>(); function getScalarsMap(program: Program): Map { diff --git a/packages/http-server-javascript/src/common/serialization/json.ts b/packages/http-server-javascript/src/common/serialization/json.ts index 6c324d3a567..ec1fa43ad4c 100644 --- a/packages/http-server-javascript/src/common/serialization/json.ts +++ b/packages/http-server-javascript/src/common/serialization/json.ts @@ -23,6 +23,7 @@ import { differentiateUnion, writeCodeTree } from "../../util/differentiate.js"; import { UnimplementedError } from "../../util/error.js"; import { indent } from "../../util/iter.js"; import { emitTypeReference, escapeUnsafeChars } from "../reference.js"; +import { getJsScalar } from "../scalar.js"; import { SerializableType, SerializationContext, requireSerialization } from "./index.js"; /** @@ -46,13 +47,20 @@ export function requiresJsonSerialization(ctx: JsContext, type: Type): boolean { switch (type.kind) { case "Model": { + if (isArrayModelType(ctx.program, type)) { + const argumentType = type.indexer.value; + requiresSerialization = requiresJsonSerialization(ctx, argumentType); + break; + } + requiresSerialization = [...type.properties.values()].some((property) => propertyRequiresJsonSerialization(ctx, property), ); break; } case "Scalar": { - requiresSerialization = getEncode(ctx.program, type) !== undefined; + const scalar = getJsScalar(ctx.program, type, type); + requiresSerialization = scalar === "Uint8Array" || getEncode(ctx.program, type) !== undefined; break; } case "Union": { @@ -104,11 +112,11 @@ export function* emitJsonSerialization( module: Module, typeName: string, ): Iterable { - yield `toJsonObject(input: ${typeName}): object {`; + yield `toJsonObject(input: ${typeName}): any {`; yield* indent(emitToJson(ctx, type, module)); yield `},`; - yield `fromJsonObject(input: object): ${typeName} {`; + yield `fromJsonObject(input: any): ${typeName} {`; yield* indent(emitFromJson(ctx, type, module)); yield `},`; } @@ -176,7 +184,7 @@ function transposeExpressionToJson( const argumentType = type.indexer.value; if (requiresJsonSerialization(ctx, argumentType)) { - return `${expr}.map((item) => ${transposeExpressionToJson(ctx, argumentType, "item", module)})`; + return `${expr}?.map((item) => ${transposeExpressionToJson(ctx, argumentType, "item", module)})`; } else { return expr; } @@ -203,7 +211,16 @@ function transposeExpressionToJson( } } case "Scalar": - return expr; + const scalar = getJsScalar(ctx.program, type, type); + + switch (scalar) { + case "Uint8Array": + // Coerce to Buffer if we aren't given a buffer. This avoids having to do unholy things to + // convert through an intermediate and use globalThis.btoa. v8 does not support Uint8Array.toBase64 + return `((${expr} instanceof Buffer) ? ${expr} : Buffer.from(${expr})).toString('base64')`; + default: + return expr; + } case "Union": if (!requiresJsonSerialization(ctx, type)) { return expr; @@ -336,7 +353,7 @@ function transposeExpressionFromJson( const argumentType = type.indexer.value; if (requiresJsonSerialization(ctx, argumentType)) { - return `${expr}.map((item) => ${transposeExpressionFromJson(ctx, argumentType, "item", module)})`; + return `${expr}?.map((item: any) => ${transposeExpressionFromJson(ctx, argumentType, "item", module)})`; } else { return expr; } @@ -363,7 +380,14 @@ function transposeExpressionFromJson( } } case "Scalar": - return expr; + const scalar = getJsScalar(ctx.program, type, type); + + switch (scalar) { + case "Uint8Array": + return `Buffer.from(${expr}, 'base64')`; + default: + return expr; + } case "Union": if (!requiresJsonSerialization(ctx, type)) { return expr; diff --git a/packages/http-server-javascript/src/helpers/multipart.ts b/packages/http-server-javascript/src/helpers/multipart.ts index 02362978a8f..53e308fccf2 100644 --- a/packages/http-server-javascript/src/helpers/multipart.ts +++ b/packages/http-server-javascript/src/helpers/multipart.ts @@ -26,6 +26,8 @@ function MultipartBoundaryTransformStream( let boundarySplit = Buffer.from(`--${boundary}`); let initialized = false; + // We need to keep at least the length of the boundary split plus room for CRLFCRLF in the buffer to detect the boundaries. + // We subtract one from this length because if the whole thing were in the buffer, we would detect it and move past it. const bufferKeepLength = boundarySplit.length + BUF_CRLFCRLF.length - 1; let _readableController: ReadableStreamDefaultController> = null as any; @@ -41,9 +43,9 @@ function MultipartBoundaryTransformStream( write: async (chunk) => { buffer = Buffer.concat([buffer, chunk]); - const index = buffer.indexOf(boundarySplit); + let index: number; - if (index !== -1) { + while ((index = buffer.indexOf(boundarySplit)) !== -1) { // We found a boundary, emit everything before it and initialize a new stream for the next part. // We are initialized if we have found the boundary at least once. @@ -88,7 +90,7 @@ function MultipartBoundaryTransformStream( } }, close() { - if (buffer.toString("utf-8") !== "--") { + if (!/--(\r\n)?/.test(buffer.toString("utf-8"))) { readableController.error(new Error("Unexpected characters after final boundary.")); } diff --git a/packages/http-server-javascript/src/http/server/index.ts b/packages/http-server-javascript/src/http/server/index.ts index d9b031d0985..947fc4d4980 100644 --- a/packages/http-server-javascript/src/http/server/index.ts +++ b/packages/http-server-javascript/src/http/server/index.ts @@ -32,6 +32,7 @@ import { differentiateUnion, writeCodeTree } from "../../util/differentiate.js"; import { emitMultipart, emitMultipartLegacy } from "./multipart.js"; import { module as headerHelpers } from "../../../generated-defs/helpers/header.js"; +import { requiresJsonSerialization } from "../../common/serialization/json.js"; const DEFAULT_CONTENT_TYPE = "application/json"; @@ -192,7 +193,7 @@ function* emitRawServerOperation( yield ` return ${names.ctx}.errorHandlers.onInvalidRequest(`; yield ` ${names.ctx},`; yield ` ${JSON.stringify(operation.path)},`; - yield ` \`unexpected "Content-Type": '\${${contentTypeHeader}?.value}', expected '${JSON.stringify(contentType)}'\``; + yield ` \`unexpected "content-type": '\${${contentTypeHeader}?.value}', expected '${JSON.stringify(contentType)}'\``; yield ` );`; yield " }"; @@ -206,7 +207,26 @@ function* emitRawServerOperation( yield ` const chunks: Array = [];`; yield ` ${names.ctx}.request.on("data", function appendChunk(chunk) { chunks.push(chunk); });`; yield ` ${names.ctx}.request.on("end", function finalize() {`; - yield ` resolve(JSON.parse(Buffer.concat(chunks).toString()));`; + yield ` try {`; + yield ` const body = Buffer.concat(chunks).toString();`; + + let value: string; + + if (requiresJsonSerialization(ctx, body.type)) { + value = `${bodyTypeName}.fromJsonObject(JSON.parse(body))`; + } else { + value = `JSON.parse(body)`; + } + + yield ` resolve(${value});`; + yield ` } catch {`; + yield ` ${names.ctx}.errorHandlers.onInvalidRequest(`; + yield ` ${names.ctx},`; + yield ` ${JSON.stringify(operation.path)},`; + yield ` "invalid JSON in request body",`; + yield ` );`; + yield ` reject();`; + yield ` }`; yield ` });`; yield ` ${names.ctx}.request.on("error", reject);`; yield ` }) as ${bodyTypeName};`; @@ -216,9 +236,11 @@ function* emitRawServerOperation( } case "multipart/form-data": if (body.bodyKind === "multipart") { - yield* indent(emitMultipart(ctx, module, operation, body, bodyName, bodyTypeName)); + yield* indent( + emitMultipart(ctx, module, operation, body, names.ctx, bodyName, bodyTypeName), + ); } else { - yield* indent(emitMultipartLegacy(bodyName, bodyTypeName)); + yield* indent(emitMultipartLegacy(names.ctx, bodyName, bodyTypeName)); } break; default: @@ -378,6 +400,9 @@ function* emitResultProcessingForType( const bodyCase = parseCase(body.name); const serializationRequired = isSerializationRequired(ctx, body.type, "application/json"); requireSerialization(ctx, body.type, "application/json"); + + yield `${names.ctx}.response.setHeader("content-type", "application/json");`; + if (serializationRequired) { const typeReference = emitTypeReference(ctx, body.type, body, module, { requireDeclaration: true, @@ -392,6 +417,9 @@ function* emitResultProcessingForType( } else { const serializationRequired = isSerializationRequired(ctx, target, "application/json"); requireSerialization(ctx, target, "application/json"); + + yield `${names.ctx}.response.setHeader("content-type", "application/json");`; + if (serializationRequired) { const typeReference = emitTypeReference(ctx, target, target, module, { requireDeclaration: true, @@ -419,6 +447,7 @@ function* emitHeaderParamBinding( parameter: Extract, ): Iterable { const nameCase = parseCase(parameter.param.name); + const headerName = parameter.name.toLowerCase(); // See https://nodejs.org/api/http.html#messageheaders // Apparently, only set-cookie can be an array. @@ -426,13 +455,12 @@ function* emitHeaderParamBinding( const assertion = canBeArrayType ? "" : " as string | undefined"; - yield `const ${nameCase.camelCase} = ${names.ctx}.request.headers[${JSON.stringify(parameter.name)}]${assertion};`; + yield `const ${nameCase.camelCase} = ${names.ctx}.request.headers[${JSON.stringify(headerName)}]${assertion};`; if (!parameter.param.optional) { yield `if (${nameCase.camelCase} === undefined) {`; // prettier-ignore - yield ` return ${names.ctx}.errorHandlers.onInvalidRequest(${names.ctx}, ${JSON.stringify(operation.path)}, "missing required header '${parameter.name}'");`; - yield ` throw new Error("Invalid request: missing required header '${parameter.name}'.");`; + yield ` return ${names.ctx}.errorHandlers.onInvalidRequest(${names.ctx}, ${JSON.stringify(operation.path)}, "missing required header '${headerName}'");`; yield "}"; yield ""; } @@ -455,7 +483,7 @@ function* emitQueryParamBinding( const nameCase = parseCase(parameter.param.name); // UrlSearchParams annoyingly returns null for missing parameters instead of undefined. - yield `const ${nameCase.camelCase} = ${names}.get(${JSON.stringify(parameter.name)}) ?? undefined;`; + yield `const ${nameCase.camelCase} = ${names.queryParams}.get(${JSON.stringify(parameter.name)}) ?? undefined;`; if (!parameter.param.optional) { yield `if (!${nameCase.camelCase}) {`; diff --git a/packages/http-server-javascript/src/http/server/multipart.ts b/packages/http-server-javascript/src/http/server/multipart.ts index 6b973f2e513..6ec51fb5279 100644 --- a/packages/http-server-javascript/src/http/server/multipart.ts +++ b/packages/http-server-javascript/src/http/server/multipart.ts @@ -4,6 +4,9 @@ import { HttpContext } from "../index.js"; import { module as headerHelpers } from "../../../generated-defs/helpers/header.js"; import { module as multipartHelpers } from "../../../generated-defs/helpers/multipart.js"; +import { emitTypeReference } from "../../common/reference.js"; +import { requireSerialization } from "../../common/serialization/index.js"; +import { requiresJsonSerialization } from "../../common/serialization/json.js"; import { parseCase } from "../../util/case.js"; import { UnimplementedError } from "../../util/error.js"; @@ -22,6 +25,7 @@ export function* emitMultipart( module: Module, operation: HttpOperation, body: HttpOperationMultipartBody, + ctxName: string, bodyName: string, bodyTypeName: string, ): Iterable { @@ -40,7 +44,7 @@ export function* emitMultipart( const stream = ctx.gensym("stream"); - yield ` const ${stream} = createMultipartReadable(request);`; + yield ` const ${stream} = createMultipartReadable(${ctxName}.request);`; yield ""; const contentDisposition = ctx.gensym("contentDisposition"); @@ -154,13 +158,30 @@ export function* emitMultipart( ); } - yield ` let __chunks = Buffer.alloc(0);`; + yield ` const __chunks = [];`; yield ""; yield ` for await (const __chunk of ${partName}.body) {`; yield ` __chunks.push(__chunk);`; yield ` }`; + + yield ` const __object = JSON.parse(Buffer.concat(__chunks).toString("utf-8"));`; yield ""; - value = 'JSON.parse(Buffer.concat(__chunks).toString("utf-8"));'; + + if (requiresJsonSerialization(ctx, namedPart.body.type)) { + const bodyTypeReference = emitTypeReference( + ctx, + namedPart.body.type, + namedPart.body.property ?? namedPart.body.type, + module, + { altName: bodyTypeName + "Body", requireDeclaration: true }, + ); + + requireSerialization(ctx, namedPart.body.type, "application/json"); + + value = `${bodyTypeReference}.fromJsonObject(__object)`; + } else { + value = "__object"; + } } if (namedPart.multi) { if (namedPart.optional) { @@ -207,16 +228,20 @@ export function* emitMultipart( // This function is old and broken. I'm not likely to fix it unless we decide to continue supporting legacy multipart // parsing after 1.0. -export function* emitMultipartLegacy(bodyName: string, bodyTypeName: string): Iterable { +export function* emitMultipartLegacy( + ctxName: string, + bodyName: string, + bodyTypeName: string, +): Iterable { yield `const ${bodyName} = await new Promise(function parse${bodyTypeName}MultipartRequest(resolve, reject) {`; - yield ` const boundary = request.headers["content-type"]?.split(";").find((s) => s.includes("boundary="))?.split("=", 2)[1];`; + yield ` const boundary = ${ctxName}.request.headers["content-type"]?.split(";").find((s) => s.includes("boundary="))?.split("=", 2)[1];`; yield ` if (!boundary) {`; yield ` return reject("Invalid request: missing boundary in content-type.");`; yield ` }`; yield ""; yield ` const chunks: Array = [];`; - yield ` request.on("data", function appendChunk(chunk) { chunks.push(chunk); });`; - yield ` request.on("end", function finalize() {`; + yield ` ${ctxName}.request.on("data", function appendChunk(chunk) { chunks.push(chunk); });`; + yield ` ${ctxName}.request.on("end", function finalize() {`; yield ` const text = Buffer.concat(chunks).toString();`; yield ` const parts = text.split(boundary).slice(1, -1);`; yield ` const fields: { [k: string]: any } = {};`; diff --git a/packages/http-server-javascript/src/http/server/router.ts b/packages/http-server-javascript/src/http/server/router.ts index 1dadb729075..3d57d029f43 100644 --- a/packages/http-server-javascript/src/http/server/router.ts +++ b/packages/http-server-javascript/src/http/server/router.ts @@ -20,7 +20,9 @@ import { bifilter, indent } from "../../util/iter.js"; import { keywordSafe } from "../../util/keywords.js"; import { HttpContext } from "../index.js"; +import { module as headerHelpers } from "../../../generated-defs/helpers/header.js"; import { module as routerHelper } from "../../../generated-defs/helpers/router.js"; +import { parseHeaderValueParameters } from "../../helpers/header.js"; import { reportDiagnostic } from "../../lib.js"; import { UnimplementedError } from "../../util/error.js"; @@ -49,6 +51,11 @@ export function emitRouter(ctx: HttpContext, service: HttpService, serverRawModu from: serverRawModule, }); + routerModule.imports.push({ + binder: ["parseHeaderValueParameters"], + from: headerHelpers, + }); + routerModule.declarations.push([...emitRouterDefinition(ctx, service, routeTree, routerModule)]); } @@ -197,7 +204,7 @@ function* emitRouterDefinition( yield ""; - yield ` return ${onRequestNotFound}(ctx);`; + yield ` return ctx.errorHandlers.onRequestNotFound(ctx);`; yield ` });`; yield ""; @@ -389,7 +396,11 @@ function* emitRouteOperationDispatchMultiple( contentTypeMap.set(operation, operationContentType.value); } - yield `switch (request.headers["content-type"]) {`; + const contentTypeName = ctx.gensym("contentType"); + + yield `const ${contentTypeName} = parseHeaderValueParameters(request.headers["content-type"])?.value;`; + + yield `switch (${contentTypeName}) {`; for (const [operation, contentType] of contentTypeMap.entries()) { const [backend] = backends.get(operation.container)!; @@ -404,12 +415,14 @@ function* emitRouteOperationDispatchMultiple( ? ", " + operation.parameters.map((param) => parseCase(param.name).camelCase).join(", ") : ""; - yield ` case ${JSON.stringify(contentType)}:`; + const contentTypeValue = parseHeaderValueParameters(contentType).value; + + yield ` case ${JSON.stringify(contentTypeValue)}:`; yield ` return ${routeHandlers}.${operationName}(ctx, ${backendMemberName}${parameters});`; } yield ` default:`; - yield ` return ctx.errorHandlers.onInvalidRequest(ctx, ${JSON.stringify(route)}, \`No operation in route '${route}' matched content-type "\${contentType}"\`);`; + yield ` return ctx.errorHandlers.onInvalidRequest(ctx, ${JSON.stringify(route)}, \`No operation in route '${route}' matched content-type "\${${contentTypeName}}"\`);`; yield "}"; } From 9d0089acacbb69f152518009616648ee91d2a212 Mon Sep 17 00:00:00 2001 From: Will Temple Date: Tue, 7 Jan 2025 11:47:18 -0500 Subject: [PATCH 3/3] Chronus --- .../witemple-msft-hsj-visibility-2025-0-7-11-43-52.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .chronus/changes/witemple-msft-hsj-visibility-2025-0-7-11-43-52.md diff --git a/.chronus/changes/witemple-msft-hsj-visibility-2025-0-7-11-43-52.md b/.chronus/changes/witemple-msft-hsj-visibility-2025-0-7-11-43-52.md new file mode 100644 index 00000000000..feefa8f83bd --- /dev/null +++ b/.chronus/changes/witemple-msft-hsj-visibility-2025-0-7-11-43-52.md @@ -0,0 +1,8 @@ +--- +changeKind: feature +packages: + - "@typespec/http-server-javascript" +--- + +- Implemented new-style multipart request handling. +- Fixed JSON serialization/deserialization in some cases where models that required serialization occurred within arrays.