Skip to content

Commit 89fd415

Browse files
sbordetsbordet
committed
Fixes #12670 - Improve buffer management of HTTP/1 response headers. (#12777)
* Backported #12544 (client-side handling of request header overflow). * Introduced HttpConfiguration.maxResponseHeaderSize * Implemented server-side handling of response header overflow, similar to the client. Signed-off-by: Simone Bordet <[email protected]>
1 parent 54b52d1 commit 89fd415

File tree

16 files changed

+202
-27
lines changed

16 files changed

+202
-27
lines changed

‎jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java

+20-1
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,9 @@ public class HttpClient extends ContainerLifeCycle implements AutoCloseable
125125
private int maxConnectionsPerDestination = 64;
126126
private int maxRequestsQueuedPerDestination = 1024;
127127
private int requestBufferSize = 4096;
128+
private int maxRequestHeadersSize = 8192;
128129
private int responseBufferSize = 16384;
130+
private int maxResponseHeadersSize = -1;
129131
private int maxRedirects = 8;
130132
private long addressResolutionTimeout = 15000;
131133
private boolean strictEventOrdering = false;
@@ -135,7 +137,6 @@ public class HttpClient extends ContainerLifeCycle implements AutoCloseable
135137
private String defaultRequestContentType = "application/octet-stream";
136138
private boolean useInputDirectByteBuffers = true;
137139
private boolean useOutputDirectByteBuffers = true;
138-
private int maxResponseHeadersSize = -1;
139140
private Sweeper destinationSweeper;
140141

141142
/**
@@ -1077,6 +1078,24 @@ public void setUseInputDirectByteBuffers(boolean useInputDirectByteBuffers)
10771078
this.useInputDirectByteBuffers = useInputDirectByteBuffers;
10781079
}
10791080

1081+
/**
1082+
* @return the max size in bytes of the request headers
1083+
*/
1084+
@ManagedAttribute("The max size in bytes of the request headers")
1085+
public int getMaxRequestHeadersSize()
1086+
{
1087+
return maxRequestHeadersSize;
1088+
}
1089+
1090+
/**
1091+
* Set the max size in bytes of the request headers.
1092+
* @param maxRequestHeadersSize the max size in bytes of the request headers
1093+
*/
1094+
public void setMaxRequestHeadersSize(int maxRequestHeadersSize)
1095+
{
1096+
this.maxRequestHeadersSize = maxRequestHeadersSize;
1097+
}
1098+
10801099
/**
10811100
* @return whether to use direct ByteBuffers for writing
10821101
*/

‎jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/internal/HttpSenderOverHTTP.java

+17-5
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ protected Action process() throws Exception
157157
HttpClient httpClient = getHttpChannel().getHttpDestination().getHttpClient();
158158
HttpExchange exchange = getHttpExchange();
159159
ByteBufferPool bufferPool = httpClient.getByteBufferPool();
160+
int requestHeadersSize = httpClient.getRequestBufferSize();
160161
boolean useDirectByteBuffers = httpClient.isUseOutputDirectByteBuffers();
161162
while (true)
162163
{
@@ -173,14 +174,25 @@ protected Action process() throws Exception
173174
{
174175
case NEED_HEADER:
175176
{
176-
headerBuffer = bufferPool.acquire(httpClient.getRequestBufferSize(), useDirectByteBuffers);
177+
generator.setMaxHeaderBytes(getHttpChannel().getHttpDestination().getHttpClient().getMaxRequestHeadersSize());
178+
headerBuffer = bufferPool.acquire(requestHeadersSize, useDirectByteBuffers);
177179
break;
178180
}
179181
case HEADER_OVERFLOW:
180182
{
181-
headerBuffer.release();
182-
headerBuffer = null;
183-
throw new IllegalArgumentException("Request header too large");
183+
int maxRequestHeadersSize = httpClient.getMaxRequestHeadersSize();
184+
if (maxRequestHeadersSize > requestHeadersSize)
185+
{
186+
generator.reset();
187+
headerBuffer.release();
188+
headerBuffer = bufferPool.acquire(maxRequestHeadersSize, useDirectByteBuffers);
189+
requestHeadersSize = maxRequestHeadersSize;
190+
break;
191+
}
192+
else
193+
{
194+
throw new IllegalArgumentException("Request headers too large");
195+
}
184196
}
185197
case NEED_CHUNK:
186198
{
@@ -189,7 +201,7 @@ protected Action process() throws Exception
189201
}
190202
case NEED_CHUNK_TRAILER:
191203
{
192-
chunkBuffer = bufferPool.acquire(httpClient.getRequestBufferSize(), useDirectByteBuffers);
204+
chunkBuffer = bufferPool.acquire(requestHeadersSize, useDirectByteBuffers);
193205
break;
194206
}
195207
case FLUSH:

‎jetty-core/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java

+105
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,11 @@
5959
import org.eclipse.jetty.io.ClientConnector;
6060
import org.eclipse.jetty.io.Content;
6161
import org.eclipse.jetty.io.EndPoint;
62+
import org.eclipse.jetty.io.RetainableByteBuffer;
6263
import org.eclipse.jetty.logging.StacklessLogging;
6364
import org.eclipse.jetty.server.Handler;
65+
import org.eclipse.jetty.server.HttpConfiguration;
66+
import org.eclipse.jetty.server.HttpConnectionFactory;
6467
import org.eclipse.jetty.server.internal.HttpChannelState;
6568
import org.eclipse.jetty.toolchain.test.Net;
6669
import org.eclipse.jetty.toolchain.test.jupiter.WorkDir;
@@ -1936,6 +1939,108 @@ protected void service(org.eclipse.jetty.server.Request request, org.eclipse.jet
19361939
assertEquals(HttpStatus.OK_200, response.getStatus());
19371940
}
19381941

1942+
@ParameterizedTest
1943+
@ArgumentsSource(ScenarioProvider.class)
1944+
public void testRequestHeadersSizeOverflow(Scenario scenario) throws Exception
1945+
{
1946+
start(scenario, new EmptyServerHandler());
1947+
1948+
RetainableByteBuffer buffer = client.getByteBufferPool().acquire(client.getRequestBufferSize(), false);
1949+
int capacity = buffer.capacity();
1950+
buffer.release();
1951+
client.setMaxRequestHeadersSize(3 * capacity);
1952+
connector.getBean(HttpConnectionFactory.class).getHttpConfiguration().setRequestHeaderSize(3 * capacity);
1953+
1954+
ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
1955+
.scheme(scenario.getScheme())
1956+
// Overflow the default request headers size, but don't exceed the max.
1957+
.agent("A".repeat(2 * capacity))
1958+
.timeout(5, TimeUnit.SECONDS)
1959+
.send();
1960+
1961+
assertEquals(HttpStatus.OK_200, response.getStatus());
1962+
}
1963+
1964+
@ParameterizedTest
1965+
@ArgumentsSource(ScenarioProvider.class)
1966+
public void testResponseHeadersSizeOverflow(Scenario scenario) throws Exception
1967+
{
1968+
start(scenario, new EmptyServerHandler()
1969+
{
1970+
@Override
1971+
protected void service(org.eclipse.jetty.server.Request request, org.eclipse.jetty.server.Response response)
1972+
{
1973+
int capacity = (int)request.getHeaders().getLongField("X-Capacity");
1974+
// Overflow the default response headers size, but don't exceed the max.
1975+
response.getHeaders().put("X-Large", "A".repeat(2 * capacity));
1976+
}
1977+
});
1978+
1979+
HttpConfiguration httpConfig = connector.getBean(HttpConnectionFactory.class).getHttpConfiguration();
1980+
RetainableByteBuffer buffer = server.getByteBufferPool().acquire(httpConfig.getResponseHeaderSize(), false);
1981+
int capacity = buffer.capacity();
1982+
buffer.release();
1983+
httpConfig.setMaxResponseHeaderSize(3 * capacity);
1984+
client.setMaxResponseHeadersSize(3 * capacity);
1985+
1986+
ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
1987+
.scheme(scenario.getScheme())
1988+
.headers(h -> h.put("X-Capacity", capacity))
1989+
.timeout(5, TimeUnit.SECONDS)
1990+
.send();
1991+
1992+
assertEquals(HttpStatus.OK_200, response.getStatus());
1993+
}
1994+
1995+
@ParameterizedTest
1996+
@ArgumentsSource(ScenarioProvider.class)
1997+
public void testMaxRequestHeadersSize(Scenario scenario) throws Exception
1998+
{
1999+
start(scenario, new EmptyServerHandler());
2000+
2001+
RetainableByteBuffer buffer = client.getByteBufferPool().acquire(client.getRequestBufferSize(), false);
2002+
int capacity = buffer.capacity();
2003+
buffer.release();
2004+
client.setMaxRequestHeadersSize(2 * capacity);
2005+
connector.getBean(HttpConnectionFactory.class).getHttpConfiguration().setRequestHeaderSize(4 * capacity);
2006+
2007+
assertThrows(ExecutionException.class, () -> client.newRequest("localhost", connector.getLocalPort())
2008+
.scheme(scenario.getScheme())
2009+
// Overflow the max request headers size.
2010+
.agent("A".repeat(3 * capacity))
2011+
.timeout(5, TimeUnit.SECONDS)
2012+
.send());
2013+
}
2014+
2015+
@ParameterizedTest
2016+
@ArgumentsSource(ScenarioProvider.class)
2017+
public void testMaxResponseHeadersSize(Scenario scenario) throws Exception
2018+
{
2019+
start(scenario, new EmptyServerHandler()
2020+
{
2021+
@Override
2022+
protected void service(org.eclipse.jetty.server.Request request, org.eclipse.jetty.server.Response response) throws Throwable
2023+
{
2024+
int capacity = (int)request.getHeaders().getLongField("X-Capacity");
2025+
// Overflow the max request headers size, should generate a 500.
2026+
response.getHeaders().put("X-Large", "A".repeat(3 * capacity));
2027+
}
2028+
});
2029+
2030+
HttpConfiguration httpConfig = connector.getBean(HttpConnectionFactory.class).getHttpConfiguration();
2031+
RetainableByteBuffer buffer = server.getByteBufferPool().acquire(httpConfig.getResponseHeaderSize(), false);
2032+
int capacity = buffer.capacity();
2033+
buffer.release();
2034+
httpConfig.setMaxResponseHeaderSize(2 * capacity);
2035+
client.setMaxResponseHeadersSize(4 * capacity);
2036+
2037+
assertThrows(ExecutionException.class, () -> client.newRequest("localhost", connector.getLocalPort())
2038+
.scheme(scenario.getScheme())
2039+
.headers(h -> h.put("X-Capacity", capacity))
2040+
.timeout(5, TimeUnit.SECONDS)
2041+
.send());
2042+
}
2043+
19392044
private void assertCopyRequest(Request original)
19402045
{
19412046
Request copy = client.copyRequest(original, original.getURI());

‎jetty-core/jetty-http2/jetty-http2-client-transport/src/main/java/org/eclipse/jetty/http2/client/transport/HttpClientTransportOverHTTP2.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ static void configure(HttpClient httpClient, HTTP2Client http2Client)
103103
http2Client.setUseOutputDirectByteBuffers(httpClient.isUseOutputDirectByteBuffers());
104104
http2Client.setConnectBlocking(httpClient.isConnectBlocking());
105105
http2Client.setBindAddress(httpClient.getBindAddress());
106-
http2Client.setMaxRequestHeadersSize(httpClient.getRequestBufferSize());
106+
http2Client.setMaxRequestHeadersSize(httpClient.getMaxRequestHeadersSize());
107107
http2Client.setMaxResponseHeadersSize(httpClient.getMaxResponseHeadersSize());
108108
}
109109

‎jetty-core/jetty-http2/jetty-http2-server/src/main/java/org/eclipse/jetty/http2/server/AbstractHTTP2ServerConnectionFactory.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,7 @@ public Connection newConnection(Connector connector, EndPoint endPoint)
306306
ServerSessionListener listener = newSessionListener(connector, endPoint);
307307

308308
Generator generator = new Generator(connector.getByteBufferPool(), isUseOutputDirectByteBuffers(), getMaxHeaderBlockFragment());
309-
generator.getHpackEncoder().setMaxHeaderListSize(getHttpConfiguration().getResponseHeaderSize());
309+
generator.getHpackEncoder().setMaxHeaderListSize(getHttpConfiguration().getMaxResponseHeaderSize());
310310

311311
FlowControlStrategy flowControl = getFlowControlStrategyFactory().newFlowControlStrategy();
312312

‎jetty-core/jetty-http2/jetty-http2-tests/src/test/java/org/eclipse/jetty/http2/tests/HttpClientTransportOverHTTP2Test.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ private void testPropertiesAreForwarded(HTTP2Client http2Client, HttpClientTrans
144144
assertEquals(httpClient.getIdleTimeout(), http2Client.getIdleTimeout());
145145
assertEquals(httpClient.isUseInputDirectByteBuffers(), http2Client.isUseInputDirectByteBuffers());
146146
assertEquals(httpClient.isUseOutputDirectByteBuffers(), http2Client.isUseOutputDirectByteBuffers());
147-
assertEquals(httpClient.getRequestBufferSize(), http2Client.getMaxRequestHeadersSize());
147+
assertEquals(httpClient.getMaxRequestHeadersSize(), http2Client.getMaxRequestHeadersSize());
148148
assertEquals(httpClient.getMaxResponseHeadersSize(), http2Client.getMaxResponseHeadersSize());
149149
}
150150
assertTrue(http2Client.isStopped());

‎jetty-core/jetty-http3/jetty-http3-client-transport/src/main/java/org/eclipse/jetty/http3/client/transport/HttpClientTransportOverHTTP3.java

+1
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ static void configure(HttpClient httpClient, HTTP3Client http3Client)
8484
configuration.setInputBufferSize(httpClient.getResponseBufferSize());
8585
configuration.setUseInputDirectByteBuffers(httpClient.isUseInputDirectByteBuffers());
8686
configuration.setUseOutputDirectByteBuffers(httpClient.isUseOutputDirectByteBuffers());
87+
configuration.setMaxRequestHeadersSize(httpClient.getMaxRequestHeadersSize());
8788
configuration.setMaxResponseHeadersSize(httpClient.getMaxResponseHeadersSize());
8889
}
8990

‎jetty-core/jetty-http3/jetty-http3-server/src/main/java/org/eclipse/jetty/http3/server/AbstractHTTP3ServerConnectionFactory.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ public AbstractHTTP3ServerConnectionFactory(ServerQuicConfiguration quicConfigur
5353
http3Configuration.setUseInputDirectByteBuffers(httpConfiguration.isUseInputDirectByteBuffers());
5454
http3Configuration.setUseOutputDirectByteBuffers(httpConfiguration.isUseOutputDirectByteBuffers());
5555
http3Configuration.setMaxRequestHeadersSize(httpConfiguration.getRequestHeaderSize());
56-
http3Configuration.setMaxResponseHeadersSize(httpConfiguration.getResponseHeaderSize());
56+
http3Configuration.setMaxResponseHeadersSize(httpConfiguration.getMaxResponseHeaderSize());
5757
}
5858

5959
public ServerQuicConfiguration getQuicConfiguration()

‎jetty-core/jetty-http3/jetty-http3-tests/src/test/java/org/eclipse/jetty/http3/tests/HttpClientTransportOverHTTP3Test.java

+1
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ private void testPropertiesAreForwarded(HTTP3Client http3Client, HttpClientTrans
9292
HTTP3Configuration http3Configuration = http3Client.getHTTP3Configuration();
9393
assertEquals(httpClient.isUseInputDirectByteBuffers(), http3Configuration.isUseInputDirectByteBuffers());
9494
assertEquals(httpClient.isUseOutputDirectByteBuffers(), http3Configuration.isUseOutputDirectByteBuffers());
95+
assertEquals(httpClient.getMaxRequestHeadersSize(), http3Configuration.getMaxRequestHeadersSize());
9596
assertEquals(httpClient.getMaxResponseHeadersSize(), http3Configuration.getMaxResponseHeadersSize());
9697
}
9798
assertTrue(http3Client.isStopped());

‎jetty-core/jetty-proxy/src/test/java/org/eclipse/jetty/proxy/ReverseProxyTest.java

+2
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ public void testServerResponseHeadersTooLargeForServerConfiguration(HttpVersion
147147

148148
int maxResponseHeadersSize = 256;
149149
serverHttpConfig.setResponseHeaderSize(maxResponseHeadersSize);
150+
serverHttpConfig.setMaxResponseHeaderSize(maxResponseHeadersSize);
150151
startServer(new Handler.Abstract()
151152
{
152153
@Override
@@ -283,6 +284,7 @@ public boolean handle(Request request, Response response, Callback callback)
283284

284285
CountDownLatch proxyToClientFailureLatch = new CountDownLatch(1);
285286
proxyHttpConfig.setResponseHeaderSize(maxResponseHeadersSize);
287+
proxyHttpConfig.setMaxResponseHeaderSize(maxResponseHeadersSize);
286288
startProxy(new ProxyHandler.Reverse(clientToProxyRequest ->
287289
HttpURI.build(clientToProxyRequest.getHttpURI()).port(serverConnector.getLocalPort()))
288290
{

‎jetty-core/jetty-server/src/main/java/org/eclipse/jetty/server/HttpConfiguration.java

+22-2
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ public class HttpConfiguration implements Dumpable
6262
private int _outputAggregationSize = _outputBufferSize / 4;
6363
private int _requestHeaderSize = 8 * 1024;
6464
private int _responseHeaderSize = 8 * 1024;
65+
private int _maxResponseHeaderSize = 16 * 1024;
6566
private int _headerCacheSize = 1024;
6667
private boolean _headerCacheCaseSensitive = false;
6768
private int _securePort;
@@ -137,6 +138,7 @@ public HttpConfiguration(HttpConfiguration config)
137138
_outputAggregationSize = config._outputAggregationSize;
138139
_requestHeaderSize = config._requestHeaderSize;
139140
_responseHeaderSize = config._responseHeaderSize;
141+
_maxResponseHeaderSize = config._maxResponseHeaderSize;
140142
_headerCacheSize = config._headerCacheSize;
141143
_headerCacheCaseSensitive = config._headerCacheCaseSensitive;
142144
_secureScheme = config._secureScheme;
@@ -217,12 +219,18 @@ public int getRequestHeaderSize()
217219
return _requestHeaderSize;
218220
}
219221

220-
@ManagedAttribute("The maximum allowed size in bytes for an HTTP response header")
222+
@ManagedAttribute("The default size in bytes for the HTTP response headers buffer")
221223
public int getResponseHeaderSize()
222224
{
223225
return _responseHeaderSize;
224226
}
225227

228+
@ManagedAttribute("The maximum size in bytes for the HTTP response headers buffer")
229+
public int getMaxResponseHeaderSize()
230+
{
231+
return _maxResponseHeaderSize;
232+
}
233+
226234
@ManagedAttribute("The maximum allowed size in Trie nodes for an HTTP header field cache")
227235
public int getHeaderCacheSize()
228236
{
@@ -442,13 +450,24 @@ public void setRequestHeaderSize(int requestHeaderSize)
442450
* <p>Larger headers will allow for more and/or larger cookies and longer HTTP headers (eg for redirection).
443451
* However, larger headers will also consume more memory.</p>
444452
*
445-
* @param responseHeaderSize the maximum size in bytes of the response header
453+
* @param responseHeaderSize the default size in bytes of the response headers buffer
446454
*/
447455
public void setResponseHeaderSize(int responseHeaderSize)
448456
{
449457
_responseHeaderSize = responseHeaderSize;
450458
}
451459

460+
/**
461+
* <p>Larger headers will allow for more and/or larger cookies and longer HTTP headers (eg for redirection).
462+
* However, larger headers will also consume more memory.</p>
463+
*
464+
* @param maxResponseHeaderSize the maximum size in bytes of the response headers buffer
465+
*/
466+
public void setMaxResponseHeaderSize(int maxResponseHeaderSize)
467+
{
468+
_maxResponseHeaderSize = maxResponseHeaderSize;
469+
}
470+
452471
/**
453472
* @param headerCacheSize The size of the header field cache, in terms of unique characters branches
454473
* in the lookup {@link Index.Mutable} and associated data structures.
@@ -839,6 +858,7 @@ public void dump(Appendable out, String indent) throws IOException
839858
"outputAggregationSize=" + _outputAggregationSize,
840859
"requestHeaderSize=" + _requestHeaderSize,
841860
"responseHeaderSize=" + _responseHeaderSize,
861+
"maxResponseHeaderSize=" + _maxResponseHeaderSize,
842862
"headerCacheSize=" + _headerCacheSize,
843863
"headerCacheCaseSensitive=" + _headerCacheCaseSensitive,
844864
"secureScheme=" + _secureScheme,

0 commit comments

Comments
 (0)