From 0e0f6af167e01f9e013da43543ea9c258a448013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=BE=D1=80=D0=B5=D0=BD=D0=B1=D0=B5=D1=80=D0=B3=20?= =?UTF-8?q?=E2=98=A2=EF=B8=8F=20=20=D0=9C=D0=B0=D1=80=D0=BA?= Date: Fri, 17 Jan 2020 19:26:20 +0500 Subject: [PATCH] Fix deflate compression (#4506) --- CHANGES/4506.bugfix | 1 + aiohttp/http_parser.py | 2 +- aiohttp/http_websocket.py | 8 +++++--- aiohttp/http_writer.py | 2 +- aiohttp/multipart.py | 4 ++-- aiohttp/web_response.py | 3 ++- tests/test_http_parser.py | 2 +- tests/test_http_writer.py | 2 +- tests/test_web_functional.py | 2 +- tests/test_web_sendfile_functional.py | 2 +- tests/test_websocket_parser.py | 2 +- 11 files changed, 17 insertions(+), 13 deletions(-) create mode 100644 CHANGES/4506.bugfix diff --git a/CHANGES/4506.bugfix b/CHANGES/4506.bugfix new file mode 100644 index 00000000000..eaf4bb88aac --- /dev/null +++ b/CHANGES/4506.bugfix @@ -0,0 +1 @@ +Fixed 'deflate' compressions. According to RFC 2616 now. diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index f2881c3bf11..978f2e1ee8b 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -727,7 +727,7 @@ def flush(self) -> bytes: self.decompressor = BrotliDecoder() # type: Any else: zlib_mode = (16 + zlib.MAX_WBITS - if encoding == 'gzip' else -zlib.MAX_WBITS) + if encoding == 'gzip' else zlib.MAX_WBITS) self.decompressor = zlib.decompressobj(wbits=zlib_mode) def set_exception(self, exc: BaseException) -> None: diff --git a/aiohttp/http_websocket.py b/aiohttp/http_websocket.py index 479bac08155..abaade91396 100644 --- a/aiohttp/http_websocket.py +++ b/aiohttp/http_websocket.py @@ -273,7 +273,7 @@ def feed_data(self, data: bytes) -> Tuple[bool, bytes]: def _feed_data(self, data: bytes) -> Tuple[bool, bytes]: for fin, opcode, payload, compressed in self.parse_frame(data): if compressed and not self._decompressobj: - self._decompressobj = zlib.decompressobj(wbits=-zlib.MAX_WBITS) + self._decompressobj = zlib.decompressobj(wbits=zlib.MAX_WBITS) if opcode == WSMsgType.CLOSE: if len(payload) >= 2: close_code = UNPACK_CLOSE_CODE(payload[:2])[0] @@ -567,11 +567,13 @@ async def _send_frame(self, message: bytes, opcode: int, # if self.compress and opcode < 8 and len(message) > 124: if (compress or self.compress) and opcode < 8: if compress: + assert compress > 0 # Do not set self._compress if compressing is for this frame - compressobj = zlib.compressobj(wbits=-compress) + compressobj = zlib.compressobj(wbits=compress) else: # self.compress if not self._compressobj: - self._compressobj = zlib.compressobj(wbits=-self.compress) + assert self.compress > 0 + self._compressobj = zlib.compressobj(wbits=self.compress) compressobj = self._compressobj message = compressobj.compress(message) diff --git a/aiohttp/http_writer.py b/aiohttp/http_writer.py index 7e27fbf6a43..102fb3ef2f4 100644 --- a/aiohttp/http_writer.py +++ b/aiohttp/http_writer.py @@ -55,7 +55,7 @@ def enable_chunking(self) -> None: def enable_compression(self, encoding: str='deflate') -> None: zlib_mode = (16 + zlib.MAX_WBITS - if encoding == 'gzip' else -zlib.MAX_WBITS) + if encoding == 'gzip' else zlib.MAX_WBITS) self._compress = zlib.compressobj(wbits=zlib_mode) def _write(self, chunk: bytes) -> None: diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 7b01bf1f742..9d78301b5f6 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -479,7 +479,7 @@ def _decode_content(self, data: bytes) -> bytes: encoding = self.headers.get(CONTENT_ENCODING, '').lower() if encoding == 'deflate': - return zlib.decompress(data, -zlib.MAX_WBITS) + return zlib.decompress(data, zlib.MAX_WBITS) elif encoding == 'gzip': return zlib.decompress(data, 16 + zlib.MAX_WBITS) elif encoding == 'identity': @@ -978,7 +978,7 @@ def enable_encoding(self, encoding: str) -> None: def enable_compression(self, encoding: str='deflate') -> None: zlib_mode = (16 + zlib.MAX_WBITS - if encoding == 'gzip' else -zlib.MAX_WBITS) + if encoding == 'gzip' else zlib.MAX_WBITS) self._compress = zlib.compressobj(wbits=zlib_mode) async def write_eof(self) -> None: diff --git a/aiohttp/web_response.py b/aiohttp/web_response.py index 4dc64976839..fdb7aa4d359 100644 --- a/aiohttp/web_response.py +++ b/aiohttp/web_response.py @@ -669,6 +669,7 @@ async def _start(self, request: 'BaseRequest') -> AbstractStreamWriter: return await super()._start(request) def _compress_body(self, zlib_mode: int) -> None: + assert zlib_mode > 0 compressobj = zlib.compressobj(wbits=zlib_mode) body_in = self._body assert body_in is not None @@ -683,7 +684,7 @@ async def _do_start_compression(self, coding: ContentCoding) -> None: # Instead of using _payload_writer.enable_compression, # compress the whole body zlib_mode = (16 + zlib.MAX_WBITS - if coding == ContentCoding.gzip else -zlib.MAX_WBITS) + if coding == ContentCoding.gzip else zlib.MAX_WBITS) body_in = self._body assert body_in is not None if self._zlib_executor_size is not None and \ diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 19fe9be7a3c..fcbf7bfe18f 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -837,7 +837,7 @@ async def test_http_payload_parser_length(self, stream) -> None: assert b'12' == b''.join(d for d, _ in out._buffer) assert b'45' == tail - _comp = zlib.compressobj(wbits=-zlib.MAX_WBITS) + _comp = zlib.compressobj(wbits=zlib.MAX_WBITS) _COMPRESSED = b''.join([_comp.compress(b'data'), _comp.flush()]) async def test_http_payload_parser_deflate(self, stream) -> None: diff --git a/tests/test_http_writer.py b/tests/test_http_writer.py index 2f8085f8a85..e948162b81a 100644 --- a/tests/test_http_writer.py +++ b/tests/test_http_writer.py @@ -117,7 +117,7 @@ async def test_write_payload_chunked_filter_mutiple_chunks( b'2\r\na2\r\n0\r\n\r\n') -compressor = zlib.compressobj(wbits=-zlib.MAX_WBITS) +compressor = zlib.compressobj(wbits=zlib.MAX_WBITS) COMPRESSED = b''.join([compressor.compress(b'data'), compressor.flush()]) diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index 4be7a962303..896c092189b 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -953,7 +953,7 @@ async def test_response_with_precompressed_body_deflate( async def handler(request): headers = {'Content-Encoding': 'deflate'} - zcomp = zlib.compressobj(wbits=-zlib.MAX_WBITS) + zcomp = zlib.compressobj(wbits=zlib.MAX_WBITS) data = zcomp.compress(b'mydata') + zcomp.flush() return web.Response(body=data, headers=headers) diff --git a/tests/test_web_sendfile_functional.py b/tests/test_web_sendfile_functional.py index c2be5dbff0d..02aceb69f7b 100644 --- a/tests/test_web_sendfile_functional.py +++ b/tests/test_web_sendfile_functional.py @@ -742,7 +742,7 @@ async def handler(request): resp = await client.get('/') assert resp.status == 200 - zcomp = zlib.compressobj(wbits=-zlib.MAX_WBITS) + zcomp = zlib.compressobj(wbits=zlib.MAX_WBITS) expected_body = zcomp.compress(b'file content\n') + zcomp.flush() assert expected_body == await resp.read() assert 'application/octet-stream' == resp.headers['Content-Type'] diff --git a/tests/test_websocket_parser.py b/tests/test_websocket_parser.py index 258a506ee92..5ee312c36dc 100644 --- a/tests/test_websocket_parser.py +++ b/tests/test_websocket_parser.py @@ -24,7 +24,7 @@ def build_frame(message, opcode, use_mask=False, noheader=False, is_fin=True, compress=False): # Send a frame over the websocket with message as its payload. if compress: - compressobj = zlib.compressobj(wbits=-9) + compressobj = zlib.compressobj(wbits=9) message = compressobj.compress(message) message = message + compressobj.flush(zlib.Z_SYNC_FLUSH) if message.endswith(_WS_DEFLATE_TRAILING):