From 90f7003f0fc41e2e8dca04336ddc92c56ee0897b Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Tue, 14 Nov 2023 17:25:47 +0900 Subject: [PATCH 01/32] add: typescript type --- types/dispatcher.d.ts | 9 +++++++-- types/header.d.ts | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/types/dispatcher.d.ts b/types/dispatcher.d.ts index 816db19d20d..baaeede8108 100644 --- a/types/dispatcher.d.ts +++ b/types/dispatcher.d.ts @@ -2,7 +2,7 @@ import { URL } from 'url' import { Duplex, Readable, Writable } from 'stream' import { EventEmitter } from 'events' import { Blob } from 'buffer' -import { IncomingHttpHeaders } from './header' +import { IncomingHttpHeaders, IncomingRawHttpHeaders } from './header' import BodyReadable from './readable' import { FormData } from './formdata' import Errors from './errors' @@ -169,12 +169,14 @@ declare namespace Dispatcher { export interface ConnectData { statusCode: number; headers: IncomingHttpHeaders; + rawHeaders: IncomingRawHttpHeaders; socket: Duplex; opaque: unknown; } export interface ResponseData { statusCode: number; headers: IncomingHttpHeaders; + rawHeaders: IncomingRawHttpHeaders; body: BodyReadable & BodyMixin; trailers: Record; opaque: unknown; @@ -183,6 +185,7 @@ declare namespace Dispatcher { export interface PipelineHandlerData { statusCode: number; headers: IncomingHttpHeaders; + rawHeaders: IncomingRawHttpHeaders; opaque: unknown; body: BodyReadable; context: object; @@ -193,12 +196,14 @@ declare namespace Dispatcher { } export interface UpgradeData { headers: IncomingHttpHeaders; + rawHeaders: IncomingRawHttpHeaders; socket: Duplex; opaque: unknown; } export interface StreamFactoryData { statusCode: number; headers: IncomingHttpHeaders; + rawHeaders: IncomingRawHttpHeaders; opaque: unknown; context: object; } @@ -211,7 +216,7 @@ declare namespace Dispatcher { /** Invoked when request is upgraded either due to a `Upgrade` header or `CONNECT` method. */ onUpgrade?(statusCode: number, headers: Buffer[] | string[] | null, socket: Duplex): void; /** Invoked when statusCode and headers have been received. May be invoked multiple times due to 1xx informational headers. */ - onHeaders?(statusCode: number, headers: Buffer[] | string[] | null, resume: () => void): boolean; + onHeaders?(statusCode: number, headers: Buffer[] | string[] | null, resume: () => void, statusText: string): boolean; /** Invoked when response payload data is received. */ onData?(chunk: Buffer): boolean; /** Invoked when response payload and trailers have been received and the request has completed. */ diff --git a/types/header.d.ts b/types/header.d.ts index bfdb3296d4d..891e162027a 100644 --- a/types/header.d.ts +++ b/types/header.d.ts @@ -2,3 +2,8 @@ * The header type declaration of `undici`. */ export type IncomingHttpHeaders = Record; + +/** + * The raw header type declaration of `undici`. + */ +export type IncomingRawHttpHeaders = { ':status'?: number } & Record; From a2c2bde5bb7210943be2103eb0ab418bf6ff248f Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Tue, 14 Nov 2023 17:29:36 +0900 Subject: [PATCH 02/32] fix: type --- types/header.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/header.d.ts b/types/header.d.ts index 891e162027a..41bb674f5e2 100644 --- a/types/header.d.ts +++ b/types/header.d.ts @@ -6,4 +6,4 @@ export type IncomingHttpHeaders = Record; /** * The raw header type declaration of `undici`. */ -export type IncomingRawHttpHeaders = { ':status'?: number } & Record; +export type IncomingRawHttpHeaders = { ':status'?: number } | Record; From d266a94b73c6552f8f0202cdc46e33a981ae051b Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Tue, 14 Nov 2023 18:12:40 +0900 Subject: [PATCH 03/32] impl --- lib/api/api-connect.js | 10 +++++---- lib/api/api-pipeline.js | 5 +++-- lib/api/api-request.js | 11 +++++----- lib/api/api-stream.js | 11 +++++----- lib/api/api-upgrade.js | 5 +++-- lib/core/util.js | 47 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 71 insertions(+), 18 deletions(-) diff --git a/lib/api/api-connect.js b/lib/api/api-connect.js index fd2b6ad97a5..eba58ea4d63 100644 --- a/lib/api/api-connect.js +++ b/lib/api/api-connect.js @@ -51,15 +51,17 @@ class ConnectHandler extends AsyncResource { this.callback = null - let headers = rawHeaders + /** @type {{ headers: any; rawHeaders: any; }} */ + let headers = { headers: {}, rawHeaders: {} } // Indicates is an HTTP2Session - if (headers != null) { - headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) + if (rawHeaders != null) { + headers = util.parseAndGetHeaders(rawHeaders, this.responseHeaders) } this.runInAsyncScope(callback, null, null, { statusCode, - headers, + headers: headers.headers, + rawHeaders: headers.rawHeaders, socket, opaque, context diff --git a/lib/api/api-pipeline.js b/lib/api/api-pipeline.js index af4a1803b44..e4dce9f0697 100644 --- a/lib/api/api-pipeline.js +++ b/lib/api/api-pipeline.js @@ -173,10 +173,11 @@ class PipelineHandler extends AsyncResource { let body try { this.handler = null - const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) + const headers = util.parseAndGetHeaders(rawHeaders, this.responseHeaders) body = this.runInAsyncScope(handler, null, { statusCode, - headers, + headers: headers.headers, + rawHeaders: headers.rawHeaders, opaque, body: this.res, context diff --git a/lib/api/api-request.js b/lib/api/api-request.js index f130ecc9867..b39af3cd44b 100644 --- a/lib/api/api-request.js +++ b/lib/api/api-request.js @@ -80,16 +80,16 @@ class RequestHandler extends AsyncResource { onHeaders (statusCode, rawHeaders, resume, statusMessage) { const { callback, opaque, abort, context, responseHeaders, highWaterMark } = this - const headers = responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) + const headers = util.parseAndGetHeaders(rawHeaders, responseHeaders) if (statusCode < 200) { if (this.onInfo) { - this.onInfo({ statusCode, headers }) + this.onInfo({ statusCode, headers: headers.rawHeaders }) } return } - const parsedHeaders = responseHeaders === 'raw' ? util.parseHeaders(rawHeaders) : headers + const parsedHeaders = responseHeaders === 'raw' ? util.parseHeaders(rawHeaders) : headers.rawHeaders const contentType = parsedHeaders['content-type'] const body = new Readable({ resume, abort, contentType, highWaterMark }) @@ -98,12 +98,13 @@ class RequestHandler extends AsyncResource { if (callback !== null) { if (this.throwOnError && statusCode >= 400) { this.runInAsyncScope(getResolveErrorBodyCallback, null, - { callback, body, contentType, statusCode, statusMessage, headers } + { callback, body, contentType, statusCode, statusMessage, headers: headers.headers, rawHeaders: headers.rawHeaders } ) } else { this.runInAsyncScope(callback, null, null, { statusCode, - headers, + headers: headers.headers, + rawHeaders: headers.rawHeaders, trailers: this.trailers, opaque, body, diff --git a/lib/api/api-stream.js b/lib/api/api-stream.js index c571a6f79a7..3a2a725a71d 100644 --- a/lib/api/api-stream.js +++ b/lib/api/api-stream.js @@ -81,11 +81,11 @@ class StreamHandler extends AsyncResource { onHeaders (statusCode, rawHeaders, resume, statusMessage) { const { factory, opaque, context, callback, responseHeaders } = this - const headers = responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) + const headers = util.parseAndGetHeaders(rawHeaders, responseHeaders) if (statusCode < 200) { if (this.onInfo) { - this.onInfo({ statusCode, headers }) + this.onInfo({ statusCode, headers: headers.rawHeaders }) } return } @@ -95,13 +95,13 @@ class StreamHandler extends AsyncResource { let res if (this.throwOnError && statusCode >= 400) { - const parsedHeaders = responseHeaders === 'raw' ? util.parseHeaders(rawHeaders) : headers + const parsedHeaders = responseHeaders === 'raw' ? util.parseHeaders(rawHeaders) : headers.rawHeaders const contentType = parsedHeaders['content-type'] res = new PassThrough() this.callback = null this.runInAsyncScope(getResolveErrorBodyCallback, null, - { callback, body: res, contentType, statusCode, statusMessage, headers } + { callback, body: res, contentType, statusCode, statusMessage, headers: headers.headers, rawHeaders: headers.rawHeaders } ) } else { if (factory === null) { @@ -110,7 +110,8 @@ class StreamHandler extends AsyncResource { res = this.runInAsyncScope(factory, null, { statusCode, - headers, + headers: headers.headers, + rawHeaders: headers.rawHeaders, opaque, context }) diff --git a/lib/api/api-upgrade.js b/lib/api/api-upgrade.js index ef783e82975..b2481b93729 100644 --- a/lib/api/api-upgrade.js +++ b/lib/api/api-upgrade.js @@ -54,9 +54,10 @@ class UpgradeHandler extends AsyncResource { removeSignal(this) this.callback = null - const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) + const headers = util.parseAndGetHeaders(rawHeaders, this.responseHeaders) this.runInAsyncScope(callback, null, null, { - headers, + headers: headers.headers, + rawHeaders: headers.rawHeaders, socket, opaque, context diff --git a/lib/core/util.js b/lib/core/util.js index 8d5450ba0c0..31fec455bcf 100644 --- a/lib/core/util.js +++ b/lib/core/util.js @@ -276,6 +276,50 @@ function parseRawHeaders (headers) { return ret } +/** + * @param {any} headers + * @returns {Record} + */ +function getHeadersFromParsedHeaders (headers) { + /** @type {Record} */ + const ret = [] + const keys = Object.keys(headers) + for (let i = 0; i < keys.length; ++i) { + const key = keys[i] + if (key.charCodeAt(0) !== 0x3a) { + ret[key] = headers[key] + } + } + return ret +} + +/** + * @param {string[]} headers + * @returns {string[]} + */ +function getHeadersFromParsedRawHeaders (headers) { + /** @type {string[]} */ + const ret = [] + for (let i = 0; i < headers.length; i += 2) { + if (headers[i].charCodeAt(0) !== 0x3a) { + ret.push(headers[i], headers[i + 1]) + } + } + return ret +} + +/** + * @param {*} rawHeaders + * @param {'raw' | null} [kind] + */ +function parseAndGetHeaders (rawHeaders, kind) { + const parsedRawHeaders = kind === 'raw' ? parseRawHeaders(rawHeaders) : parseHeaders(rawHeaders) + return { + headers: kind === 'raw' ? getHeadersFromParsedRawHeaders(parsedRawHeaders) : getHeadersFromParsedHeaders(parsedRawHeaders), + rawHeaders: parsedRawHeaders + } +} + function isBuffer (buffer) { // See, https://github.com/mcollina/undici/pull/319 return buffer instanceof Uint8Array || Buffer.isBuffer(buffer) @@ -491,6 +535,9 @@ module.exports = { isDestroyed, parseRawHeaders, parseHeaders, + getHeadersFromParsedHeaders, + getHeadersFromParsedRawHeaders, + parseAndGetHeaders, parseKeepAliveTimeout, destroy, bodyLength, From 5de62026b0ba9f931d88aa0850451ff347a6b693 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Tue, 14 Nov 2023 18:14:31 +0900 Subject: [PATCH 04/32] fix --- lib/core/util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/util.js b/lib/core/util.js index 31fec455bcf..ec2c48c7191 100644 --- a/lib/core/util.js +++ b/lib/core/util.js @@ -282,7 +282,7 @@ function parseRawHeaders (headers) { */ function getHeadersFromParsedHeaders (headers) { /** @type {Record} */ - const ret = [] + const ret = {} const keys = Object.keys(headers) for (let i = 0; i < keys.length; ++i) { const key = keys[i] From 2c89ffb6490ed226d2079db99cc416f61303b977 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Tue, 14 Nov 2023 18:37:47 +0900 Subject: [PATCH 05/32] fix: h2 fetch --- lib/fetch/index.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/fetch/index.js b/lib/fetch/index.js index c109a01bf1f..d88859ba70d 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -1994,19 +1994,24 @@ async function httpNetworkFetch ( for (let n = 0; n < headersList.length; n += 2) { const key = headersList[n + 0].toString('latin1') const val = headersList[n + 1].toString('latin1') - if (key.toLowerCase() === 'content-encoding') { + if (key.charCodeAt(0) === 0x3a) { + // Starts with a colon is a pseudo-header. e.g. `:status` + // Colons are not allowed in the fetch spec and should be removed. + continue + } else if (key.toLowerCase() === 'content-encoding') { // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1 // "All content-coding values are case-insensitive..." - codings = val.toLowerCase().split(',').map((x) => x.trim()) - } else if (key.toLowerCase() === 'location') { - location = val - } + codings = val.toLowerCase().split(',').map((x) => x.trim()) + } else if (key.toLowerCase() === 'location') { + location = val + } headers[kHeadersList].append(key, val) } } else { const keys = Object.keys(headersList) - for (const key of keys) { + for (let i = 0; i < keys.length; ++i) { + const key = keys[i] const val = headersList[key] if (key.toLowerCase() === 'content-encoding') { // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1 From fad90883ba369e1410cf668fbbc31ebb809caa12 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:19:25 +0900 Subject: [PATCH 06/32] fix: lint --- lib/fetch/index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/fetch/index.js b/lib/fetch/index.js index d88859ba70d..a5c38dbe0a0 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -2001,10 +2001,10 @@ async function httpNetworkFetch ( } else if (key.toLowerCase() === 'content-encoding') { // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1 // "All content-coding values are case-insensitive..." - codings = val.toLowerCase().split(',').map((x) => x.trim()) - } else if (key.toLowerCase() === 'location') { - location = val - } + codings = val.toLowerCase().split(',').map((x) => x.trim()) + } else if (key.toLowerCase() === 'location') { + location = val + } headers[kHeadersList].append(key, val) } From 438162e50fa62bfb4ef45a1ab26162429bc23b59 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:32:09 +0900 Subject: [PATCH 07/32] test: h2 fetch --- test/fetch/issue-2415.js | 52 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 test/fetch/issue-2415.js diff --git a/test/fetch/issue-2415.js b/test/fetch/issue-2415.js new file mode 100644 index 00000000000..a638bd41772 --- /dev/null +++ b/test/fetch/issue-2415.js @@ -0,0 +1,52 @@ +'use strict' +const { createSecureServer } = require('node:http2') +const { once } = require('node:events') + +const pem = require('https-pem') + +const { test } = require('tap') +const { Client, fetch } = require('../..') + +test('Issue#2415', async (t) => { + const server = createSecureServer(pem) + + server.on('stream', async (stream, headers) => { + stream.respond({ + ':status': 200 + }) + stream.end('test') + }) + + server.listen() + await once(server, 'listening') + + const client = new Client(`https://localhost:${server.address().port}`, { + connect: { + rejectUnauthorized: false + }, + allowH2: true + }) + + const response = await fetch( + `https://localhost:${server.address().port}/`, + // Needs to be passed to disable the reject unauthorized + { + method: 'GET', + dispatcher: client + } + ) + + await response.text() + + t.teardown(server.close.bind(server)) + t.teardown(client.close.bind(client)) + + for (const key of response.headers.keys()) { + t.notOk( + key.startsWith(':'), + `The pseudo-headers \`${key}\` must not be included in \`Headers#keys\`.` + ) + } + + t.end() +}) From ea6693fa56958002097e7f52c52cb9b5cc3aa76a Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:51:02 +0900 Subject: [PATCH 08/32] fix: h2 fetch --- lib/fetch/index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/fetch/index.js b/lib/fetch/index.js index a5c38dbe0a0..6340116fe68 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -2013,7 +2013,11 @@ async function httpNetworkFetch ( for (let i = 0; i < keys.length; ++i) { const key = keys[i] const val = headersList[key] - if (key.toLowerCase() === 'content-encoding') { + if (key.charCodeAt(0) === 0x3a) { + // Starts with a colon is a pseudo-header. e.g. `:status` + // Colons are not allowed in the fetch spec and should be removed. + continue + } else if (key.toLowerCase() === 'content-encoding') { // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1 // "All content-coding values are case-insensitive..." codings = val.toLowerCase().split(',').map((x) => x.trim()).reverse() From b7b4245dc84d2ad17785123104ae67dd812fcfdc Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Tue, 14 Nov 2023 21:42:08 +0900 Subject: [PATCH 09/32] test: more test --- test/util.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/util.js b/test/util.js index 794c68e3f77..678cc9772ef 100644 --- a/test/util.js +++ b/test/util.js @@ -121,3 +121,19 @@ test('buildURL', { skip: util.nodeMajor >= 12 }, (t) => { t.end() }) + +test('getHeadersFromParsedHeaders', t => { + t.plan(1) + t.same(util.getHeadersFromParsedHeaders({ ':status': 200, host: 'localhost' }), { host: 'localhost' }) +}) + +test('getHeadersFromParsedRawHeaders', t => { + t.plan(1) + t.same(util.getHeadersFromParsedRawHeaders([':status', 200, 'host', 'localhost']), ['host', 'localhost']) +}) + +test('parseAndGetHeaders', t => { + t.plan(2) + t.same(util.parseAndGetHeaders({ ':status': 200, host: 'localhost' }, null), { headers: { host: 'localhost' }, rawHeaders: { ':status': 200, host: 'localhost' } }) + t.same(util.parseAndGetHeaders(['host', 'localhost'], 'raw'), { headers: ['host', 'localhost'], rawHeaders: ['host', 'localhost'] }) +}) From 3b161234972cfdd6d9d01ecce3f5805fcc062d8d Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Tue, 14 Nov 2023 21:50:49 +0900 Subject: [PATCH 10/32] test: simple --- test/fetch/http2.js | 40 ++++++++++++++++++++++++++++++- test/fetch/issue-2415.js | 52 ---------------------------------------- 2 files changed, 39 insertions(+), 53 deletions(-) delete mode 100644 test/fetch/issue-2415.js diff --git a/test/fetch/http2.js b/test/fetch/http2.js index 9b3c95b8f1f..f726ea5e30e 100644 --- a/test/fetch/http2.js +++ b/test/fetch/http2.js @@ -13,7 +13,7 @@ const { Client, fetch } = require('../..') const nodeVersion = Number(process.version.split('v')[1].split('.')[0]) -plan(6) +plan(7) test('[Fetch] Issue#2311', async t => { const expectedBody = 'hello from client!' @@ -375,3 +375,41 @@ test( t.equal(Buffer.concat(requestChunks).toString('utf-8'), expectedBody) } ) + +test('Issue#2415', async (t) => { + t.plan(1) + const server = createSecureServer(pem) + + server.on('stream', async (stream, headers) => { + stream.respond({ + ':status': 200 + }) + stream.end('test') + }) + + server.listen() + await once(server, 'listening') + + const client = new Client(`https://localhost:${server.address().port}`, { + connect: { + rejectUnauthorized: false + }, + allowH2: true + }) + + const response = await fetch( + `https://localhost:${server.address().port}/`, + // Needs to be passed to disable the reject unauthorized + { + method: 'GET', + dispatcher: client + } + ) + + await response.text() + + t.teardown(server.close.bind(server)) + t.teardown(client.close.bind(client)) + + t.doesNotThrow(() => new Headers(response.headers)) +}) diff --git a/test/fetch/issue-2415.js b/test/fetch/issue-2415.js deleted file mode 100644 index a638bd41772..00000000000 --- a/test/fetch/issue-2415.js +++ /dev/null @@ -1,52 +0,0 @@ -'use strict' -const { createSecureServer } = require('node:http2') -const { once } = require('node:events') - -const pem = require('https-pem') - -const { test } = require('tap') -const { Client, fetch } = require('../..') - -test('Issue#2415', async (t) => { - const server = createSecureServer(pem) - - server.on('stream', async (stream, headers) => { - stream.respond({ - ':status': 200 - }) - stream.end('test') - }) - - server.listen() - await once(server, 'listening') - - const client = new Client(`https://localhost:${server.address().port}`, { - connect: { - rejectUnauthorized: false - }, - allowH2: true - }) - - const response = await fetch( - `https://localhost:${server.address().port}/`, - // Needs to be passed to disable the reject unauthorized - { - method: 'GET', - dispatcher: client - } - ) - - await response.text() - - t.teardown(server.close.bind(server)) - t.teardown(client.close.bind(client)) - - for (const key of response.headers.keys()) { - t.notOk( - key.startsWith(':'), - `The pseudo-headers \`${key}\` must not be included in \`Headers#keys\`.` - ) - } - - t.end() -}) From ce4173b0a73ea414f818a71eee719985d853e094 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 06:31:01 +0900 Subject: [PATCH 11/32] revet --- lib/api/api-connect.js | 10 ++++------ lib/api/api-pipeline.js | 5 ++--- lib/api/api-request.js | 11 +++++------ lib/api/api-stream.js | 11 +++++------ lib/api/api-upgrade.js | 5 ++--- 5 files changed, 18 insertions(+), 24 deletions(-) diff --git a/lib/api/api-connect.js b/lib/api/api-connect.js index eba58ea4d63..fd2b6ad97a5 100644 --- a/lib/api/api-connect.js +++ b/lib/api/api-connect.js @@ -51,17 +51,15 @@ class ConnectHandler extends AsyncResource { this.callback = null - /** @type {{ headers: any; rawHeaders: any; }} */ - let headers = { headers: {}, rawHeaders: {} } + let headers = rawHeaders // Indicates is an HTTP2Session - if (rawHeaders != null) { - headers = util.parseAndGetHeaders(rawHeaders, this.responseHeaders) + if (headers != null) { + headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) } this.runInAsyncScope(callback, null, null, { statusCode, - headers: headers.headers, - rawHeaders: headers.rawHeaders, + headers, socket, opaque, context diff --git a/lib/api/api-pipeline.js b/lib/api/api-pipeline.js index e4dce9f0697..af4a1803b44 100644 --- a/lib/api/api-pipeline.js +++ b/lib/api/api-pipeline.js @@ -173,11 +173,10 @@ class PipelineHandler extends AsyncResource { let body try { this.handler = null - const headers = util.parseAndGetHeaders(rawHeaders, this.responseHeaders) + const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) body = this.runInAsyncScope(handler, null, { statusCode, - headers: headers.headers, - rawHeaders: headers.rawHeaders, + headers, opaque, body: this.res, context diff --git a/lib/api/api-request.js b/lib/api/api-request.js index b39af3cd44b..f130ecc9867 100644 --- a/lib/api/api-request.js +++ b/lib/api/api-request.js @@ -80,16 +80,16 @@ class RequestHandler extends AsyncResource { onHeaders (statusCode, rawHeaders, resume, statusMessage) { const { callback, opaque, abort, context, responseHeaders, highWaterMark } = this - const headers = util.parseAndGetHeaders(rawHeaders, responseHeaders) + const headers = responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) if (statusCode < 200) { if (this.onInfo) { - this.onInfo({ statusCode, headers: headers.rawHeaders }) + this.onInfo({ statusCode, headers }) } return } - const parsedHeaders = responseHeaders === 'raw' ? util.parseHeaders(rawHeaders) : headers.rawHeaders + const parsedHeaders = responseHeaders === 'raw' ? util.parseHeaders(rawHeaders) : headers const contentType = parsedHeaders['content-type'] const body = new Readable({ resume, abort, contentType, highWaterMark }) @@ -98,13 +98,12 @@ class RequestHandler extends AsyncResource { if (callback !== null) { if (this.throwOnError && statusCode >= 400) { this.runInAsyncScope(getResolveErrorBodyCallback, null, - { callback, body, contentType, statusCode, statusMessage, headers: headers.headers, rawHeaders: headers.rawHeaders } + { callback, body, contentType, statusCode, statusMessage, headers } ) } else { this.runInAsyncScope(callback, null, null, { statusCode, - headers: headers.headers, - rawHeaders: headers.rawHeaders, + headers, trailers: this.trailers, opaque, body, diff --git a/lib/api/api-stream.js b/lib/api/api-stream.js index 3a2a725a71d..c571a6f79a7 100644 --- a/lib/api/api-stream.js +++ b/lib/api/api-stream.js @@ -81,11 +81,11 @@ class StreamHandler extends AsyncResource { onHeaders (statusCode, rawHeaders, resume, statusMessage) { const { factory, opaque, context, callback, responseHeaders } = this - const headers = util.parseAndGetHeaders(rawHeaders, responseHeaders) + const headers = responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) if (statusCode < 200) { if (this.onInfo) { - this.onInfo({ statusCode, headers: headers.rawHeaders }) + this.onInfo({ statusCode, headers }) } return } @@ -95,13 +95,13 @@ class StreamHandler extends AsyncResource { let res if (this.throwOnError && statusCode >= 400) { - const parsedHeaders = responseHeaders === 'raw' ? util.parseHeaders(rawHeaders) : headers.rawHeaders + const parsedHeaders = responseHeaders === 'raw' ? util.parseHeaders(rawHeaders) : headers const contentType = parsedHeaders['content-type'] res = new PassThrough() this.callback = null this.runInAsyncScope(getResolveErrorBodyCallback, null, - { callback, body: res, contentType, statusCode, statusMessage, headers: headers.headers, rawHeaders: headers.rawHeaders } + { callback, body: res, contentType, statusCode, statusMessage, headers } ) } else { if (factory === null) { @@ -110,8 +110,7 @@ class StreamHandler extends AsyncResource { res = this.runInAsyncScope(factory, null, { statusCode, - headers: headers.headers, - rawHeaders: headers.rawHeaders, + headers, opaque, context }) diff --git a/lib/api/api-upgrade.js b/lib/api/api-upgrade.js index b2481b93729..ef783e82975 100644 --- a/lib/api/api-upgrade.js +++ b/lib/api/api-upgrade.js @@ -54,10 +54,9 @@ class UpgradeHandler extends AsyncResource { removeSignal(this) this.callback = null - const headers = util.parseAndGetHeaders(rawHeaders, this.responseHeaders) + const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) this.runInAsyncScope(callback, null, null, { - headers: headers.headers, - rawHeaders: headers.rawHeaders, + headers, socket, opaque, context From 232f4893cb8c3643291c2c5030e4fe42e1fcf58d Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 06:32:25 +0900 Subject: [PATCH 12/32] revert --- lib/core/util.js | 27 --------------------------- test/util.js | 11 ----------- 2 files changed, 38 deletions(-) diff --git a/lib/core/util.js b/lib/core/util.js index ec2c48c7191..545d7cea53b 100644 --- a/lib/core/util.js +++ b/lib/core/util.js @@ -293,33 +293,6 @@ function getHeadersFromParsedHeaders (headers) { return ret } -/** - * @param {string[]} headers - * @returns {string[]} - */ -function getHeadersFromParsedRawHeaders (headers) { - /** @type {string[]} */ - const ret = [] - for (let i = 0; i < headers.length; i += 2) { - if (headers[i].charCodeAt(0) !== 0x3a) { - ret.push(headers[i], headers[i + 1]) - } - } - return ret -} - -/** - * @param {*} rawHeaders - * @param {'raw' | null} [kind] - */ -function parseAndGetHeaders (rawHeaders, kind) { - const parsedRawHeaders = kind === 'raw' ? parseRawHeaders(rawHeaders) : parseHeaders(rawHeaders) - return { - headers: kind === 'raw' ? getHeadersFromParsedRawHeaders(parsedRawHeaders) : getHeadersFromParsedHeaders(parsedRawHeaders), - rawHeaders: parsedRawHeaders - } -} - function isBuffer (buffer) { // See, https://github.com/mcollina/undici/pull/319 return buffer instanceof Uint8Array || Buffer.isBuffer(buffer) diff --git a/test/util.js b/test/util.js index 678cc9772ef..ee933305b4c 100644 --- a/test/util.js +++ b/test/util.js @@ -126,14 +126,3 @@ test('getHeadersFromParsedHeaders', t => { t.plan(1) t.same(util.getHeadersFromParsedHeaders({ ':status': 200, host: 'localhost' }), { host: 'localhost' }) }) - -test('getHeadersFromParsedRawHeaders', t => { - t.plan(1) - t.same(util.getHeadersFromParsedRawHeaders([':status', 200, 'host', 'localhost']), ['host', 'localhost']) -}) - -test('parseAndGetHeaders', t => { - t.plan(2) - t.same(util.parseAndGetHeaders({ ':status': 200, host: 'localhost' }, null), { headers: { host: 'localhost' }, rawHeaders: { ':status': 200, host: 'localhost' } }) - t.same(util.parseAndGetHeaders(['host', 'localhost'], 'raw'), { headers: ['host', 'localhost'], rawHeaders: ['host', 'localhost'] }) -}) From 5b24d6bff00d5795c46a97a2ce9636927478ea54 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 06:41:29 +0900 Subject: [PATCH 13/32] change --- lib/core/util.js | 6 +++--- test/util.js | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/core/util.js b/lib/core/util.js index 545d7cea53b..6f40ecd78a4 100644 --- a/lib/core/util.js +++ b/lib/core/util.js @@ -277,10 +277,10 @@ function parseRawHeaders (headers) { } /** - * @param {any} headers - * @returns {Record} + * @param {Record} headers + * @returns {Record} */ -function getHeadersFromParsedHeaders (headers) { +function omitPseudoHeaders (headers) { /** @type {Record} */ const ret = {} const keys = Object.keys(headers) diff --git a/test/util.js b/test/util.js index ee933305b4c..d0bee4ba87c 100644 --- a/test/util.js +++ b/test/util.js @@ -122,7 +122,7 @@ test('buildURL', { skip: util.nodeMajor >= 12 }, (t) => { t.end() }) -test('getHeadersFromParsedHeaders', t => { +test('omitPseudoHeaders', t => { t.plan(1) - t.same(util.getHeadersFromParsedHeaders({ ':status': 200, host: 'localhost' }), { host: 'localhost' }) + t.same(util.omitPseudoHeaders({ ':status': 200, host: 'localhost' }), { host: 'localhost' }) }) From 64d8f6c3f01c0e3a04979cb180a69cc1a8bfdcdc Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 06:42:00 +0900 Subject: [PATCH 14/32] simple type --- lib/core/util.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/core/util.js b/lib/core/util.js index 6f40ecd78a4..8904cd8c730 100644 --- a/lib/core/util.js +++ b/lib/core/util.js @@ -278,10 +278,9 @@ function parseRawHeaders (headers) { /** * @param {Record} headers - * @returns {Record} */ function omitPseudoHeaders (headers) { - /** @type {Record} */ + /** @type {Record} */ const ret = {} const keys = Object.keys(headers) for (let i = 0; i < keys.length; ++i) { From cb567d618e9313a60d582de542aa0c39c49f6e02 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 09:28:36 +0900 Subject: [PATCH 15/32] add: type --- types/dispatcher.d.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/types/dispatcher.d.ts b/types/dispatcher.d.ts index baaeede8108..f9812c8ee5f 100644 --- a/types/dispatcher.d.ts +++ b/types/dispatcher.d.ts @@ -215,8 +215,12 @@ declare namespace Dispatcher { onError?(err: Error): void; /** Invoked when request is upgraded either due to a `Upgrade` header or `CONNECT` method. */ onUpgrade?(statusCode: number, headers: Buffer[] | string[] | null, socket: Duplex): void; - /** Invoked when statusCode and headers have been received. May be invoked multiple times due to 1xx informational headers. */ - onHeaders?(statusCode: number, headers: Buffer[] | string[] | null, resume: () => void, statusText: string): boolean; + /** + * Invoked when statusCode and headers have been received. May be invoked multiple times due to 1xx informational headers. + * + * If h2, it is rawHeaders instead of statusText, if http/1.1 it remains as string. + */ + onHeaders?(statusCode: number, headers: Buffer[] | string[] | null, resume: () => void, statusTextOrRawHeaders: string | IncomingRawHttpHeaders): boolean; /** Invoked when response payload data is received. */ onData?(chunk: Buffer): boolean; /** Invoked when response payload and trailers have been received and the request has completed. */ From 4167e4c0c4cbfa044713ed82d40bf2ed915f5965 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 09:29:53 +0900 Subject: [PATCH 16/32] fix --- lib/core/util.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/core/util.js b/lib/core/util.js index 8904cd8c730..de305e93f04 100644 --- a/lib/core/util.js +++ b/lib/core/util.js @@ -277,7 +277,7 @@ function parseRawHeaders (headers) { } /** - * @param {Record} headers + * @param {Record} headers */ function omitPseudoHeaders (headers) { /** @type {Record} */ @@ -507,9 +507,7 @@ module.exports = { isDestroyed, parseRawHeaders, parseHeaders, - getHeadersFromParsedHeaders, - getHeadersFromParsedRawHeaders, - parseAndGetHeaders, + omitPseudoHeaders, parseKeepAliveTimeout, destroy, bodyLength, From 3651728b84edfe6822a1fa8e27d4577620513697 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 09:35:05 +0900 Subject: [PATCH 17/32] better type --- types/header.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/header.d.ts b/types/header.d.ts index 41bb674f5e2..0c6b99b8d41 100644 --- a/types/header.d.ts +++ b/types/header.d.ts @@ -6,4 +6,4 @@ export type IncomingHttpHeaders = Record; /** * The raw header type declaration of `undici`. */ -export type IncomingRawHttpHeaders = { ':status'?: number } | Record; +export type IncomingRawHttpHeaders = { ':status'?: number } | Record; From 9031b089332adb676d3d641fd88130d7e995f9b9 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 09:35:32 +0900 Subject: [PATCH 18/32] better type --- types/header.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/header.d.ts b/types/header.d.ts index 0c6b99b8d41..3a5cd8bf2aa 100644 --- a/types/header.d.ts +++ b/types/header.d.ts @@ -6,4 +6,4 @@ export type IncomingHttpHeaders = Record; /** * The raw header type declaration of `undici`. */ -export type IncomingRawHttpHeaders = { ':status'?: number } | Record; +export type IncomingRawHttpHeaders = Record; From aff36ab54127aab085679904866c2fb3a92b2651 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 09:45:14 +0900 Subject: [PATCH 19/32] impl --- lib/client.js | 7 ++----- lib/fetch/index.js | 16 ++++------------ 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/lib/client.js b/lib/client.js index 70001247553..241315c4dd8 100644 --- a/lib/client.js +++ b/lib/client.js @@ -1682,6 +1682,7 @@ function writeH2 (client, session, request) { return false } + /** @type {import('node:http2').ClientHttp2Stream} */ let stream const h2State = client[kHTTP2SessionState] @@ -1777,14 +1778,10 @@ function writeH2 (client, session, request) { const shouldEndStream = method === 'GET' || method === 'HEAD' if (expectContinue) { headers[HTTP2_HEADER_EXPECT] = '100-continue' - /** - * @type {import('node:http2').ClientHttp2Stream} - */ stream = session.request(headers, { endStream: shouldEndStream, signal }) stream.once('continue', writeBodyH2) } else { - /** @type {import('node:http2').ClientHttp2Stream} */ stream = session.request(headers, { endStream: shouldEndStream, signal @@ -1796,7 +1793,7 @@ function writeH2 (client, session, request) { ++h2State.openStreams stream.once('response', headers => { - if (request.onHeaders(Number(headers[HTTP2_HEADER_STATUS]), headers, stream.resume.bind(stream), '') === false) { + if (request.onHeaders(Number(headers[HTTP2_HEADER_STATUS]), util.omitPseudoHeaders(headers), stream.resume.bind(stream), headers) === false) { stream.pause() } }) diff --git a/lib/fetch/index.js b/lib/fetch/index.js index 6340116fe68..48c1a325ab7 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -1978,7 +1978,7 @@ async function httpNetworkFetch ( } }, - onHeaders (status, headersList, resume, statusText) { + onHeaders (status, headersList, resume, statusTextOrRawHeaders) { if (status < 200) { return } @@ -1994,11 +1994,7 @@ async function httpNetworkFetch ( for (let n = 0; n < headersList.length; n += 2) { const key = headersList[n + 0].toString('latin1') const val = headersList[n + 1].toString('latin1') - if (key.charCodeAt(0) === 0x3a) { - // Starts with a colon is a pseudo-header. e.g. `:status` - // Colons are not allowed in the fetch spec and should be removed. - continue - } else if (key.toLowerCase() === 'content-encoding') { + if (key.toLowerCase() === 'content-encoding') { // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1 // "All content-coding values are case-insensitive..." codings = val.toLowerCase().split(',').map((x) => x.trim()) @@ -2013,11 +2009,7 @@ async function httpNetworkFetch ( for (let i = 0; i < keys.length; ++i) { const key = keys[i] const val = headersList[key] - if (key.charCodeAt(0) === 0x3a) { - // Starts with a colon is a pseudo-header. e.g. `:status` - // Colons are not allowed in the fetch spec and should be removed. - continue - } else if (key.toLowerCase() === 'content-encoding') { + if (key.toLowerCase() === 'content-encoding') { // https://www.rfc-editor.org/rfc/rfc7231#section-3.1.2.1 // "All content-coding values are case-insensitive..." codings = val.toLowerCase().split(',').map((x) => x.trim()).reverse() @@ -2063,7 +2055,7 @@ async function httpNetworkFetch ( resolve({ status, - statusText, + statusText: typeof statusTextOrRawHeaders === "string" ? statusTextOrRawHeaders : "", headersList: headers[kHeadersList], body: decoders.length ? pipeline(this.body, ...decoders, () => { }) From 455a4265e8096a2cde4de9f83bc1065f0bfc0821 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:35:53 +0900 Subject: [PATCH 20/32] impl & fix --- lib/api/api-pipeline.js | 3 ++- lib/api/api-request.js | 5 +++-- lib/api/api-stream.js | 5 +++-- lib/fetch/index.js | 2 +- types/dispatcher.d.ts | 2 -- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/api/api-pipeline.js b/lib/api/api-pipeline.js index af4a1803b44..d2f89bf67cc 100644 --- a/lib/api/api-pipeline.js +++ b/lib/api/api-pipeline.js @@ -157,7 +157,7 @@ class PipelineHandler extends AsyncResource { this.context = context } - onHeaders (statusCode, rawHeaders, resume) { + onHeaders (statusCode, rawHeaders, resume, statusMessageOrRawHeaders) { const { opaque, handler, context } = this if (statusCode < 200) { @@ -177,6 +177,7 @@ class PipelineHandler extends AsyncResource { body = this.runInAsyncScope(handler, null, { statusCode, headers, + rawHeaders: typeof statusMessageOrRawHeaders === 'string' ? headers : statusMessageOrRawHeaders, opaque, body: this.res, context diff --git a/lib/api/api-request.js b/lib/api/api-request.js index f130ecc9867..eca58d729af 100644 --- a/lib/api/api-request.js +++ b/lib/api/api-request.js @@ -77,7 +77,7 @@ class RequestHandler extends AsyncResource { this.context = context } - onHeaders (statusCode, rawHeaders, resume, statusMessage) { + onHeaders (statusCode, rawHeaders, resume, statusMessageOrRawHeaders) { const { callback, opaque, abort, context, responseHeaders, highWaterMark } = this const headers = responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) @@ -98,12 +98,13 @@ class RequestHandler extends AsyncResource { if (callback !== null) { if (this.throwOnError && statusCode >= 400) { this.runInAsyncScope(getResolveErrorBodyCallback, null, - { callback, body, contentType, statusCode, statusMessage, headers } + { callback, body, contentType, statusCode, statusMessage: typeof statusMessageOrRawHeaders === 'string' ? statusMessageOrRawHeaders : '', headers } ) } else { this.runInAsyncScope(callback, null, null, { statusCode, headers, + rawHeaders: typeof statusMessageOrRawHeaders === 'string' ? headers : statusMessageOrRawHeaders, trailers: this.trailers, opaque, body, diff --git a/lib/api/api-stream.js b/lib/api/api-stream.js index c571a6f79a7..b2bd131a300 100644 --- a/lib/api/api-stream.js +++ b/lib/api/api-stream.js @@ -78,7 +78,7 @@ class StreamHandler extends AsyncResource { this.context = context } - onHeaders (statusCode, rawHeaders, resume, statusMessage) { + onHeaders (statusCode, rawHeaders, resume, statusMessageOrRawHeaders) { const { factory, opaque, context, callback, responseHeaders } = this const headers = responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) @@ -101,7 +101,7 @@ class StreamHandler extends AsyncResource { this.callback = null this.runInAsyncScope(getResolveErrorBodyCallback, null, - { callback, body: res, contentType, statusCode, statusMessage, headers } + { callback, body: res, contentType, statusCode, statusMessage: typeof statusMessageOrRawHeaders === 'string' ? statusMessageOrRawHeaders : '', headers } ) } else { if (factory === null) { @@ -111,6 +111,7 @@ class StreamHandler extends AsyncResource { res = this.runInAsyncScope(factory, null, { statusCode, headers, + rawHeaders: typeof statusMessageOrRawHeaders === 'string' ? headers : statusMessageOrRawHeaders, opaque, context }) diff --git a/lib/fetch/index.js b/lib/fetch/index.js index 48c1a325ab7..a8680b0351c 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -2055,7 +2055,7 @@ async function httpNetworkFetch ( resolve({ status, - statusText: typeof statusTextOrRawHeaders === "string" ? statusTextOrRawHeaders : "", + statusText: typeof statusTextOrRawHeaders === 'string' ? statusTextOrRawHeaders : '', headersList: headers[kHeadersList], body: decoders.length ? pipeline(this.body, ...decoders, () => { }) diff --git a/types/dispatcher.d.ts b/types/dispatcher.d.ts index f9812c8ee5f..f00ddacfacd 100644 --- a/types/dispatcher.d.ts +++ b/types/dispatcher.d.ts @@ -169,7 +169,6 @@ declare namespace Dispatcher { export interface ConnectData { statusCode: number; headers: IncomingHttpHeaders; - rawHeaders: IncomingRawHttpHeaders; socket: Duplex; opaque: unknown; } @@ -196,7 +195,6 @@ declare namespace Dispatcher { } export interface UpgradeData { headers: IncomingHttpHeaders; - rawHeaders: IncomingRawHttpHeaders; socket: Duplex; opaque: unknown; } From ab7bf56fb2645ca047a9a56291ec25432b99fd51 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 19:04:39 +0900 Subject: [PATCH 21/32] doc: add --- docs/api/Dispatcher.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/api/Dispatcher.md b/docs/api/Dispatcher.md index fd463bfea16..e9b512507b3 100644 --- a/docs/api/Dispatcher.md +++ b/docs/api/Dispatcher.md @@ -209,7 +209,7 @@ Returns: `Boolean` - `false` if dispatcher is busy and further dispatch calls wo * **onConnect** `(abort: () => void, context: object) => void` - Invoked before request is dispatched on socket. May be invoked multiple times when a request is retried when the request at the head of the pipeline fails. * **onError** `(error: Error) => void` - Invoked when an error has occurred. May not throw. * **onUpgrade** `(statusCode: number, headers: Buffer[], socket: Duplex) => void` (optional) - Invoked when request is upgraded. Required if `DispatchOptions.upgrade` is defined or `DispatchOptions.method === 'CONNECT'`. -* **onHeaders** `(statusCode: number, headers: Buffer[], resume: () => void, statusText: string) => boolean` - Invoked when statusCode and headers have been received. May be invoked multiple times due to 1xx informational headers. Not required for `upgrade` requests. +* **onHeaders** `(statusCode: number, headers: Buffer[], resume: () => void, statusTextOrRawHeaders: string) => boolean` - Invoked when statusCode and headers have been received. May be invoked multiple times due to 1xx informational headers. If h2, it is rawHeaders instead of statusText, if http/1.1 it remains as string. Not required for `upgrade` requests. * **onData** `(chunk: Buffer) => boolean` - Invoked when response payload data is received. Not required for `upgrade` requests. * **onComplete** `(trailers: Buffer[]) => void` - Invoked when response payload and trailers have been received and the request has completed. Not required for `upgrade` requests. * **onBodySent** `(chunk: string | Buffer | Uint8Array) => void` - Invoked when a body chunk is sent to the server. Not required. For a stream or iterable body this will be invoked for every chunk. For other body types, it will be invoked once after the body is sent. @@ -385,6 +385,7 @@ Extends: [`RequestOptions`](#parameter-requestoptions) * **statusCode** `number` * **headers** `Record` +* **rawHeaders** `Record` * **opaque** `unknown` * **body** `stream.Readable` * **context** `object` @@ -479,6 +480,7 @@ The `RequestOptions.method` property should not be value `'CONNECT'`. * **statusCode** `number` * **headers** `Record` - Note that all header keys are lower-cased, e. g. `content-type`. +* **rawHeaders** `Record` - Almost the same as **headers**, but in the case of h2 there is a pseudo-headers. * **body** `stream.Readable` which also implements [the body mixin from the Fetch Standard](https://fetch.spec.whatwg.org/#body-mixin). * **trailers** `Record` - This object starts out as empty and will be mutated to contain trailers after `body` has emitted `'end'`. @@ -646,6 +648,7 @@ Returns: `void | Promise` - Only returns a `Promise` if no `callback * **statusCode** `number` * **headers** `Record` +* **rawHeaders** `Record` * **opaque** `unknown` * **onInfo** `({statusCode: number, headers: Record}) => void | null` (optional) - Default: `null` - Callback collecting all the info headers (HTTP 100-199) received. From a14dabf60523a5d770e71c115cfbb220e4f41e5f Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 19:06:41 +0900 Subject: [PATCH 22/32] doc: more add --- docs/api/Dispatcher.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/Dispatcher.md b/docs/api/Dispatcher.md index e9b512507b3..8b2018d2f45 100644 --- a/docs/api/Dispatcher.md +++ b/docs/api/Dispatcher.md @@ -385,7 +385,7 @@ Extends: [`RequestOptions`](#parameter-requestoptions) * **statusCode** `number` * **headers** `Record` -* **rawHeaders** `Record` +* **rawHeaders** `Record` - Almost the same as **headers**, but in the case of h2 there is a pseudo-headers. * **opaque** `unknown` * **body** `stream.Readable` * **context** `object` @@ -648,7 +648,7 @@ Returns: `void | Promise` - Only returns a `Promise` if no `callback * **statusCode** `number` * **headers** `Record` -* **rawHeaders** `Record` +* **rawHeaders** `Record`- Almost the same as **headers**, but in the case of h2 there is a pseudo-headers. * **opaque** `unknown` * **onInfo** `({statusCode: number, headers: Record}) => void | null` (optional) - Default: `null` - Callback collecting all the info headers (HTTP 100-199) received. From 28780f4dfc278a5932209a1989848dd5a3b4b6ac Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 22:00:58 +0900 Subject: [PATCH 23/32] test: add --- test/client-pipeline.js | 28 ++++++++++++++++++++++++++++ test/client-request.js | 24 ++++++++++++++++++++++++ test/client-stream.js | 25 +++++++++++++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/test/client-pipeline.js b/test/client-pipeline.js index 9b677a02774..496deb3c763 100644 --- a/test/client-pipeline.js +++ b/test/client-pipeline.js @@ -1040,3 +1040,31 @@ test('pipeline abort after headers', (t) => { }) }) }) + +test('rawHeaders must be equal to headers', (t) => { + t.plan(2) + const server = createServer(async (req, res) => { + res.writeHead(200, { 'content-type': 'text/plain', 'x-powered-by': 'NodeJS' }) + res.end() + }) + + t.teardown(server.close.bind(server)) + + server.listen(0, async () => { + const client = new Client(`http://localhost:${server.address().port}`) + t.teardown(client.destroy.bind(client)) + + client.pipeline({ + path: '/', + method: 'GET' + }, ({ body, headers, rawHeaders }) => { + t.equal(rawHeaders, headers) + return body + }).end() + .on('data', () => {}) + .on('end', () => {}) + .on('close', () => { + t.pass() + }) + }) +}) diff --git a/test/client-request.js b/test/client-request.js index 3e6670523b8..9092837b29d 100644 --- a/test/client-request.js +++ b/test/client-request.js @@ -995,3 +995,27 @@ test('request post body DataView', (t) => { t.pass() }) }) + +test('rawHeaders must be equal to headers', (t) => { + t.plan(1) + const server = createServer(async (req, res) => { + res.writeHead(200, { 'content-type': 'text/plain', 'x-powered-by': 'NodeJS' }) + res.end() + }) + + t.teardown(server.close.bind(server)) + + server.listen(0, async () => { + const client = new Client(`http://localhost:${server.address().port}`) + t.teardown(client.destroy.bind(client)) + + const res = await client.request({ + path: '/', + method: 'GET', + maxRedirections: 2 + }) + + await res.body.text() + t.equal(res.rawHeaders, res.headers) + }) +}) diff --git a/test/client-stream.js b/test/client-stream.js index a230c443b67..80962dd5d48 100644 --- a/test/client-stream.js +++ b/test/client-stream.js @@ -845,3 +845,28 @@ test('stream legacy needDrain', (t) => { }) }) }) + +test('rawHeaders must be equal to headers', (t) => { + t.plan(2) + const server = createServer(async (req, res) => { + res.writeHead(200, { 'content-type': 'text/plain', 'x-powered-by': 'NodeJS' }) + res.end() + }) + + t.teardown(server.close.bind(server)) + + server.listen(0, async () => { + const client = new Client(`http://localhost:${server.address().port}`) + t.teardown(client.destroy.bind(client)) + + client.stream({ + path: '/', + method: 'GET', + opaque: [] + }, (data) => { + t.equal(data.rawHeaders, data.headers) + }, () => { + t.pass() + }) + }) +}) From 6ac164d7b3ac93dd0a6e1bd11de2a21fff73143d Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 22:05:46 +0900 Subject: [PATCH 24/32] impl onInfo --- lib/api/api-request.js | 2 +- types/dispatcher.d.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/api/api-request.js b/lib/api/api-request.js index eca58d729af..3fec61d0eff 100644 --- a/lib/api/api-request.js +++ b/lib/api/api-request.js @@ -84,7 +84,7 @@ class RequestHandler extends AsyncResource { if (statusCode < 200) { if (this.onInfo) { - this.onInfo({ statusCode, headers }) + this.onInfo({ statusCode, headers, rawHeaders: typeof statusMessageOrRawHeaders === 'string' ? headers : statusMessageOrRawHeaders }) } return } diff --git a/types/dispatcher.d.ts b/types/dispatcher.d.ts index f00ddacfacd..d0814263656 100644 --- a/types/dispatcher.d.ts +++ b/types/dispatcher.d.ts @@ -141,7 +141,7 @@ declare namespace Dispatcher { /** Default: 0 */ maxRedirections?: number; /** Default: `null` */ - onInfo?: (info: { statusCode: number, headers: Record }) => void; + onInfo?: (info: { statusCode: number, headers: Record, rawHeaders: Record }) => void; /** Default: `null` */ responseHeader?: 'raw' | null; /** Default: `64 KiB` */ From b86b7507da5854df7d3e4ae84fcffa53681c9192 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 22:13:51 +0900 Subject: [PATCH 25/32] test: simple --- test/client-request.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/client-request.js b/test/client-request.js index 9092837b29d..22ae09c4c53 100644 --- a/test/client-request.js +++ b/test/client-request.js @@ -1012,7 +1012,6 @@ test('rawHeaders must be equal to headers', (t) => { const res = await client.request({ path: '/', method: 'GET', - maxRedirections: 2 }) await res.body.text() From aad49b8fda9eac529c31a594fa8beafc4ca1b925 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 22:14:29 +0900 Subject: [PATCH 26/32] fix: lint --- test/client-request.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/client-request.js b/test/client-request.js index 22ae09c4c53..4d32292687e 100644 --- a/test/client-request.js +++ b/test/client-request.js @@ -1011,7 +1011,7 @@ test('rawHeaders must be equal to headers', (t) => { const res = await client.request({ path: '/', - method: 'GET', + method: 'GET' }) await res.body.text() From 19d5b6397c76e98dd13a79422e9b01a240d69f44 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 22:23:58 +0900 Subject: [PATCH 27/32] test: add h2 --- test/http2.js | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/test/http2.js b/test/http2.js index a3d24f23670..0bb4562cb89 100644 --- a/test/http2.js +++ b/test/http2.js @@ -17,7 +17,7 @@ const isGreaterThanv20 = gte(process.version.slice(1), '20.0.0') // https://github.com/nodejs/node/pull/41735 const hasPseudoHeadersOrderFix = gte(process.version.slice(1), '16.14.1') -plan(22) +plan(23) test('Should support H2 connection', async t => { const body = [] @@ -1154,3 +1154,40 @@ test( t.equal(response.statusCode, 200) } ) + +test('The h2 pseudo-headers is not included in the header.', async t => { + const server = createSecureServer(pem) + + server.on('stream', (stream, headers) => { + stream.respond({ + ':status': 200 + }) + stream.end('hello h2!') + }) + + server.listen(0) + await once(server, 'listening') + + const client = new Agent({ + connect: { + rejectUnauthorized: false + }, + allowH2: true + }) + + t.plan(3) + t.teardown(server.close.bind(server)) + t.teardown(client.close.bind(client)) + + const response = await client.request({ + origin: `https://localhost:${server.address().port}`, + path: '/', + method: 'GET' + }) + + await response.body.text() + + t.equal(response.statusCode, 200) + t.equal(response.headers[':status'], undefined) + t.equal(response.rawHeaders[':status'], 200) +}) From f74b950fc996980cb2e91ec47fb1ffa8a265dd1f Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 22:31:08 +0900 Subject: [PATCH 28/32] test: rename --- test/http2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/http2.js b/test/http2.js index 0bb4562cb89..7ec8b3740b4 100644 --- a/test/http2.js +++ b/test/http2.js @@ -1155,7 +1155,7 @@ test( } ) -test('The h2 pseudo-headers is not included in the header.', async t => { +test('The h2 pseudo-headers is not included in the header', async t => { const server = createSecureServer(pem) server.on('stream', (stream, headers) => { From 93d9f0c143c2d6e81907aba8215b8c3299427233 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 22:36:07 +0900 Subject: [PATCH 29/32] fix: missing import --- test/fetch/http2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fetch/http2.js b/test/fetch/http2.js index f726ea5e30e..9f6997f821b 100644 --- a/test/fetch/http2.js +++ b/test/fetch/http2.js @@ -9,7 +9,7 @@ const { Readable } = require('node:stream') const { test, plan } = require('tap') const pem = require('https-pem') -const { Client, fetch } = require('../..') +const { Client, fetch, Headers } = require('../..') const nodeVersion = Number(process.version.split('v')[1].split('.')[0]) From 87d3448c03dd95763f58485a8614b5e3797f17fc Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Wed, 15 Nov 2023 22:52:24 +0900 Subject: [PATCH 30/32] fix: missing --- lib/api/api-pipeline.js | 2 +- lib/api/api-stream.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/api/api-pipeline.js b/lib/api/api-pipeline.js index d2f89bf67cc..2ad5a271e29 100644 --- a/lib/api/api-pipeline.js +++ b/lib/api/api-pipeline.js @@ -163,7 +163,7 @@ class PipelineHandler extends AsyncResource { if (statusCode < 200) { if (this.onInfo) { const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) - this.onInfo({ statusCode, headers }) + this.onInfo({ statusCode, headers, rawHeaders: typeof statusMessageOrRawHeaders === 'string' ? headers : statusMessageOrRawHeaders }) } return } diff --git a/lib/api/api-stream.js b/lib/api/api-stream.js index b2bd131a300..ece42a50f36 100644 --- a/lib/api/api-stream.js +++ b/lib/api/api-stream.js @@ -85,7 +85,7 @@ class StreamHandler extends AsyncResource { if (statusCode < 200) { if (this.onInfo) { - this.onInfo({ statusCode, headers }) + this.onInfo({ statusCode, headers, rawHeaders: typeof statusMessageOrRawHeaders === 'string' ? headers : statusMessageOrRawHeaders }) } return } From 41385607dc572f53db9fc03be4d0587b66c620a1 Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Thu, 16 Nov 2023 06:41:17 +0900 Subject: [PATCH 31/32] rename --- lib/api/api-pipeline.js | 6 +++--- lib/api/api-request.js | 6 +++--- lib/api/api-stream.js | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/api/api-pipeline.js b/lib/api/api-pipeline.js index 2ad5a271e29..4f28719f025 100644 --- a/lib/api/api-pipeline.js +++ b/lib/api/api-pipeline.js @@ -157,12 +157,12 @@ class PipelineHandler extends AsyncResource { this.context = context } - onHeaders (statusCode, rawHeaders, resume, statusMessageOrRawHeaders) { + onHeaders (statusCode, headersList, resume, statusMessageOrRawHeaders) { const { opaque, handler, context } = this if (statusCode < 200) { if (this.onInfo) { - const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) + const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(headersList) : util.parseHeaders(headersList) this.onInfo({ statusCode, headers, rawHeaders: typeof statusMessageOrRawHeaders === 'string' ? headers : statusMessageOrRawHeaders }) } return @@ -173,7 +173,7 @@ class PipelineHandler extends AsyncResource { let body try { this.handler = null - const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) + const headers = this.responseHeaders === 'raw' ? util.parseRawHeaders(headersList) : util.parseHeaders(headersList) body = this.runInAsyncScope(handler, null, { statusCode, headers, diff --git a/lib/api/api-request.js b/lib/api/api-request.js index 3fec61d0eff..c6c7a1eaaa5 100644 --- a/lib/api/api-request.js +++ b/lib/api/api-request.js @@ -77,10 +77,10 @@ class RequestHandler extends AsyncResource { this.context = context } - onHeaders (statusCode, rawHeaders, resume, statusMessageOrRawHeaders) { + onHeaders (statusCode, headersList, resume, statusMessageOrRawHeaders) { const { callback, opaque, abort, context, responseHeaders, highWaterMark } = this - const headers = responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) + const headers = responseHeaders === 'raw' ? util.parseRawHeaders(headersList) : util.parseHeaders(headersList) if (statusCode < 200) { if (this.onInfo) { @@ -89,7 +89,7 @@ class RequestHandler extends AsyncResource { return } - const parsedHeaders = responseHeaders === 'raw' ? util.parseHeaders(rawHeaders) : headers + const parsedHeaders = responseHeaders === 'raw' ? util.parseHeaders(headersList) : headers const contentType = parsedHeaders['content-type'] const body = new Readable({ resume, abort, contentType, highWaterMark }) diff --git a/lib/api/api-stream.js b/lib/api/api-stream.js index ece42a50f36..c44ea0329eb 100644 --- a/lib/api/api-stream.js +++ b/lib/api/api-stream.js @@ -78,10 +78,10 @@ class StreamHandler extends AsyncResource { this.context = context } - onHeaders (statusCode, rawHeaders, resume, statusMessageOrRawHeaders) { + onHeaders (statusCode, headersList, resume, statusMessageOrRawHeaders) { const { factory, opaque, context, callback, responseHeaders } = this - const headers = responseHeaders === 'raw' ? util.parseRawHeaders(rawHeaders) : util.parseHeaders(rawHeaders) + const headers = responseHeaders === 'raw' ? util.parseRawHeaders(headersList) : util.parseHeaders(headersList) if (statusCode < 200) { if (this.onInfo) { @@ -95,7 +95,7 @@ class StreamHandler extends AsyncResource { let res if (this.throwOnError && statusCode >= 400) { - const parsedHeaders = responseHeaders === 'raw' ? util.parseHeaders(rawHeaders) : headers + const parsedHeaders = responseHeaders === 'raw' ? util.parseHeaders(headersList) : headers const contentType = parsedHeaders['content-type'] res = new PassThrough() From 2c9904cd0ee4db35aeedfd7e364f7357c341938a Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Thu, 16 Nov 2023 13:17:30 +0900 Subject: [PATCH 32/32] fix: test --- test/http2.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/http2.js b/test/http2.js index 7ec8b3740b4..b180c325224 100644 --- a/test/http2.js +++ b/test/http2.js @@ -1168,7 +1168,7 @@ test('The h2 pseudo-headers is not included in the header', async t => { server.listen(0) await once(server, 'listening') - const client = new Agent({ + const client = new Client(`https://localhost:${server.address().port}`, { connect: { rejectUnauthorized: false }, @@ -1180,7 +1180,6 @@ test('The h2 pseudo-headers is not included in the header', async t => { t.teardown(client.close.bind(client)) const response = await client.request({ - origin: `https://localhost:${server.address().port}`, path: '/', method: 'GET' })