diff --git a/CHANGES/4056.feature b/CHANGES/4056.feature new file mode 100644 index 00000000000..b7fa18b268d --- /dev/null +++ b/CHANGES/4056.feature @@ -0,0 +1 @@ +Compat Python 3.8. diff --git a/aiohttp/base_protocol.py b/aiohttp/base_protocol.py index cf4f9dad2ea..331241d9bc0 100644 --- a/aiohttp/base_protocol.py +++ b/aiohttp/base_protocol.py @@ -6,7 +6,7 @@ class BaseProtocol(asyncio.Protocol): __slots__ = ('_loop', '_paused', '_drain_waiter', - '_connection_lost', 'transport') + '_connection_lost', '_reading_paused', 'transport') def __init__(self, loop: asyncio.AbstractEventLoop) -> None: self._loop = loop # type: asyncio.AbstractEventLoop diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index ce1d8bfe497..a38a3bb01c9 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -53,6 +53,7 @@ PY_36 = sys.version_info >= (3, 6) PY_37 = sys.version_info >= (3, 7) +PY_38 = sys.version_info >= (3, 8) if not PY_37: import idna_ssl diff --git a/aiohttp/streams.py b/aiohttp/streams.py index 8b1d020bd6d..352f452a321 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -13,7 +13,6 @@ except ImportError: from typing_extensions import Deque # noqa - __all__ = ( 'EMPTY_PAYLOAD', 'EofStream', 'StreamReader', 'DataQueue', 'FlowControlDataQueue') @@ -409,7 +408,7 @@ async def readexactly(self, n: int) -> bytes: block = await self.read(n) if not block: partial = b''.join(blocks) - raise asyncio.streams.IncompleteReadError( + raise asyncio.IncompleteReadError( partial, len(partial) + n) blocks.append(block) n -= len(block) @@ -514,7 +513,7 @@ async def readchunk(self) -> Tuple[bytes, bool]: return (b'', True) async def readexactly(self, n: int) -> bytes: - raise asyncio.streams.IncompleteReadError(b'', n) + raise asyncio.IncompleteReadError(b'', n) def read_nowait(self) -> bytes: return b'' diff --git a/aiohttp/test_utils.py b/aiohttp/test_utils.py index 70323986297..48e10b80d1a 100644 --- a/aiohttp/test_utils.py +++ b/aiohttp/test_utils.py @@ -4,6 +4,7 @@ import contextlib import functools import gc +import inspect import socket import sys import unittest @@ -634,10 +635,11 @@ def make_mocked_request(method: str, path: str, def make_mocked_coro(return_value: Any=sentinel, raise_exception: Any=sentinel) -> Any: """Creates a coroutine mock.""" - @asyncio.coroutine - def mock_coro(*args: Any, **kwargs: Any) -> Any: + async def mock_coro(*args: Any, **kwargs: Any) -> Any: if raise_exception is not sentinel: raise raise_exception - return return_value + if not inspect.isawaitable(return_value): + return return_value + await return_value return mock.Mock(wraps=mock_coro) diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index 765d2e7debb..374d01809d0 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -2420,8 +2420,7 @@ async def handler(request): await aiohttp.request('GET', server.make_url('/')) -@asyncio.coroutine -def test_yield_from_in_session_request(aiohttp_client) -> None: +async def test_yield_from_in_session_request(aiohttp_client) -> None: # a test for backward compatibility with yield from syntax async def handler(request): return web.Response() @@ -2429,13 +2428,12 @@ async def handler(request): app = web.Application() app.router.add_get('/', handler) - client = yield from aiohttp_client(app) - resp = yield from client.get('/') + client = await aiohttp_client(app) + resp = await client.get('/') assert resp.status == 200 -@asyncio.coroutine -def test_close_context_manager(aiohttp_client) -> None: +async def test_close_context_manager(aiohttp_client) -> None: # a test for backward compatibility with yield from syntax async def handler(request): return web.Response() @@ -2443,7 +2441,7 @@ async def handler(request): app = web.Application() app.router.add_get('/', handler) - client = yield from aiohttp_client(app) + client = await aiohttp_client(app) ctx = client.get('/') ctx.close() assert not ctx._coro.cr_running diff --git a/tests/test_client_request.py b/tests/test_client_request.py index 7b5882de29f..df22f8ca494 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -917,7 +917,7 @@ async def gen(): assert req.headers['TRANSFER-ENCODING'] == 'chunked' async def throw_exc(): - await asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01) fut.set_exception(ValueError) loop.create_task(throw_exc()) @@ -942,7 +942,7 @@ async def gen(): inner_exc = ValueError() async def throw_exc(): - await asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01) fut.set_exception(inner_exc) loop.create_task(throw_exc()) @@ -970,7 +970,7 @@ async def gen(): assert req.chunked async def coro(): - await asyncio.sleep(0.0001, loop=loop) + await asyncio.sleep(0.0001) req._continue.set_result(1) loop.create_task(coro()) @@ -989,7 +989,7 @@ async def test_data_continue(loop, buf, conn) -> None: expect100=True, loop=loop) async def coro(): - await asyncio.sleep(0.0001, loop=loop) + await asyncio.sleep(0.0001) req._continue.set_result(1) loop.create_task(coro()) diff --git a/tests/test_client_session.py b/tests/test_client_session.py index 0a2a51a9594..23a78c8b321 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -17,6 +17,7 @@ from aiohttp.client_reqrep import ClientRequest from aiohttp.connector import BaseConnector, TCPConnector from aiohttp.helpers import PY_36 +from aiohttp.test_utils import make_mocked_coro @pytest.fixture @@ -164,7 +165,11 @@ async def test_merge_headers_with_list_of_tuples_duplicated_names( def test_http_GET(session, params) -> None: - with mock.patch("aiohttp.client.ClientSession._request") as patched: + # Python 3.8 will auto use mock.AsyncMock, it has different behavior + with mock.patch( + "aiohttp.client.ClientSession._request", + new_callable=mock.MagicMock + ) as patched: session.get("http://test.example.com", params={"x": 1}, **params) @@ -177,7 +182,10 @@ def test_http_GET(session, params) -> None: def test_http_OPTIONS(session, params) -> None: - with mock.patch("aiohttp.client.ClientSession._request") as patched: + with mock.patch( + "aiohttp.client.ClientSession._request", + new_callable=mock.MagicMock + ) as patched: session.options("http://opt.example.com", params={"x": 2}, **params) @@ -190,7 +198,10 @@ def test_http_OPTIONS(session, params) -> None: def test_http_HEAD(session, params) -> None: - with mock.patch("aiohttp.client.ClientSession._request") as patched: + with mock.patch( + "aiohttp.client.ClientSession._request", + new_callable=mock.MagicMock + ) as patched: session.head("http://head.example.com", params={"x": 2}, **params) @@ -203,7 +214,10 @@ def test_http_HEAD(session, params) -> None: def test_http_POST(session, params) -> None: - with mock.patch("aiohttp.client.ClientSession._request") as patched: + with mock.patch( + "aiohttp.client.ClientSession._request", + new_callable=mock.MagicMock + ) as patched: session.post("http://post.example.com", params={"x": 2}, data="Some_data", @@ -217,7 +231,10 @@ def test_http_POST(session, params) -> None: def test_http_PUT(session, params) -> None: - with mock.patch("aiohttp.client.ClientSession._request") as patched: + with mock.patch( + "aiohttp.client.ClientSession._request", + new_callable=mock.MagicMock + ) as patched: session.put("http://put.example.com", params={"x": 2}, data="Some_data", @@ -231,7 +248,10 @@ def test_http_PUT(session, params) -> None: def test_http_PATCH(session, params) -> None: - with mock.patch("aiohttp.client.ClientSession._request") as patched: + with mock.patch( + "aiohttp.client.ClientSession._request", + new_callable=mock.MagicMock + ) as patched: session.patch("http://patch.example.com", params={"x": 2}, data="Some_data", @@ -245,7 +265,10 @@ def test_http_PATCH(session, params) -> None: def test_http_DELETE(session, params) -> None: - with mock.patch("aiohttp.client.ClientSession._request") as patched: + with mock.patch( + "aiohttp.client.ClientSession._request", + new_callable=mock.MagicMock + ) as patched: session.delete("http://delete.example.com", params={"x": 2}, **params) @@ -491,7 +514,10 @@ async def test_session_default_version(loop) -> None: def test_proxy_str(session, params) -> None: - with mock.patch("aiohttp.client.ClientSession._request") as patched: + with mock.patch( + "aiohttp.client.ClientSession._request", + new_callable=mock.MagicMock + ) as patched: session.get("http://test.example.com", proxy='http://proxy.com', **params) @@ -515,9 +541,9 @@ async def handler(request): body = 'This is request body' gathered_req_body = BytesIO() gathered_res_body = BytesIO() - on_request_start = mock.Mock(side_effect=asyncio.coroutine(mock.Mock())) - on_request_redirect = mock.Mock(side_effect=asyncio.coroutine(mock.Mock())) - on_request_end = mock.Mock(side_effect=asyncio.coroutine(mock.Mock())) + on_request_start = mock.Mock(side_effect=make_mocked_coro(mock.Mock())) + on_request_redirect = mock.Mock(side_effect=make_mocked_coro(mock.Mock())) + on_request_end = mock.Mock(side_effect=make_mocked_coro(mock.Mock())) async def on_request_chunk_sent(session, context, params): gathered_req_body.write(params.chunk) @@ -568,9 +594,9 @@ async def on_response_chunk_received(session, context, params): async def test_request_tracing_exception(loop) -> None: - on_request_end = mock.Mock(side_effect=asyncio.coroutine(mock.Mock())) + on_request_end = mock.Mock(side_effect=make_mocked_coro(mock.Mock())) on_request_exception = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock()) + side_effect=make_mocked_coro(mock.Mock()) ) trace_config = aiohttp.TraceConfig() diff --git a/tests/test_connector.py b/tests/test_connector.py index 03f5d8bd830..e138beb674b 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -734,7 +734,7 @@ async def test_tcp_connector_dns_throttle_requests_cancelled_when_close( await asyncio.sleep(0) await conn.close() - with pytest.raises(asyncio.futures.CancelledError): + with pytest.raises(asyncio.CancelledError): await f @@ -742,16 +742,16 @@ async def test_tcp_connector_dns_tracing(loop, dns_response) -> None: session = mock.Mock() trace_config_ctx = mock.Mock() on_dns_resolvehost_start = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock()) + side_effect=make_mocked_coro(mock.Mock()) ) on_dns_resolvehost_end = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock()) + side_effect=make_mocked_coro(mock.Mock()) ) on_dns_cache_hit = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock()) + side_effect=make_mocked_coro(mock.Mock()) ) on_dns_cache_miss = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock()) + side_effect=make_mocked_coro(mock.Mock()) ) trace_config = aiohttp.TraceConfig( @@ -817,10 +817,10 @@ async def test_tcp_connector_dns_tracing_cache_disabled(loop, session = mock.Mock() trace_config_ctx = mock.Mock() on_dns_resolvehost_start = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock()) + side_effect=make_mocked_coro(mock.Mock()) ) on_dns_resolvehost_end = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock()) + side_effect=make_mocked_coro(mock.Mock()) ) trace_config = aiohttp.TraceConfig( @@ -891,10 +891,10 @@ async def test_tcp_connector_dns_tracing_throttle_requests( session = mock.Mock() trace_config_ctx = mock.Mock() on_dns_cache_hit = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock()) + side_effect=make_mocked_coro(mock.Mock()) ) on_dns_cache_miss = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock()) + side_effect=make_mocked_coro(mock.Mock()) ) trace_config = aiohttp.TraceConfig( @@ -1029,10 +1029,10 @@ async def test_connect_tracing(loop) -> None: session = mock.Mock() trace_config_ctx = mock.Mock() on_connection_create_start = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock()) + side_effect=make_mocked_coro(mock.Mock()) ) on_connection_create_end = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock()) + side_effect=make_mocked_coro(mock.Mock()) ) trace_config = aiohttp.TraceConfig( @@ -1421,10 +1421,10 @@ async def test_connect_queued_operation_tracing(loop, key) -> None: session = mock.Mock() trace_config_ctx = mock.Mock() on_connection_queued_start = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock()) + side_effect=make_mocked_coro(mock.Mock()) ) on_connection_queued_end = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock()) + side_effect=make_mocked_coro(mock.Mock()) ) trace_config = aiohttp.TraceConfig( @@ -1481,7 +1481,7 @@ async def test_connect_reuseconn_tracing(loop, key) -> None: session = mock.Mock() trace_config_ctx = mock.Mock() on_connection_reuseconn = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock()) + side_effect=make_mocked_coro(mock.Mock()) ) trace_config = aiohttp.TraceConfig( diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 864498ec3cd..ca6059cb539 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -344,7 +344,7 @@ def test_timer_context_no_task(loop) -> None: async def test_weakref_handle(loop) -> None: cb = mock.Mock() helpers.weakref_handle(cb, 'test', 0.01, loop, False) - await asyncio.sleep(0.1, loop=loop) + await asyncio.sleep(0.1) assert cb.test.called @@ -353,7 +353,7 @@ async def test_weakref_handle_weak(loop) -> None: helpers.weakref_handle(cb, 'test', 0.01, loop, False) del cb gc.collect() - await asyncio.sleep(0.1, loop=loop) + await asyncio.sleep(0.1) def test_ceil_call_later() -> None: diff --git a/tests/test_proxy.py b/tests/test_proxy.py index 5a16a300191..da1d6d30f51 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -224,8 +224,7 @@ async def make_conn(): seq = 0 - @asyncio.coroutine - def create_connection(*args, **kwargs): + async def create_connection(*args, **kwargs): nonlocal seq seq += 1 @@ -275,8 +274,7 @@ async def make_conn(): seq = 0 - @asyncio.coroutine - def create_connection(*args, **kwargs): + async def create_connection(*args, **kwargs): nonlocal seq seq += 1 diff --git a/tests/test_tracing.py b/tests/test_tracing.py index 71488afd108..1155f2539dd 100644 --- a/tests/test_tracing.py +++ b/tests/test_tracing.py @@ -1,9 +1,9 @@ -import asyncio from types import SimpleNamespace from unittest.mock import Mock import pytest +from aiohttp.test_utils import make_mocked_coro from aiohttp.tracing import ( Trace, TraceConfig, @@ -145,7 +145,7 @@ class TestTrace: async def test_send(self, signal, params, param_obj) -> None: session = Mock() trace_request_ctx = Mock() - callback = Mock(side_effect=asyncio.coroutine(Mock())) + callback = Mock(side_effect=make_mocked_coro(Mock())) trace_config = TraceConfig() getattr(trace_config, "on_%s" % signal).append(callback) diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index 81b60642ce0..7212fd334cb 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -20,6 +20,7 @@ multipart, web, ) +from aiohttp.test_utils import make_mocked_coro try: import ssl @@ -1752,17 +1753,17 @@ async def handler(request): async def test_request_tracing(aiohttp_server) -> None: - on_request_start = mock.Mock(side_effect=asyncio.coroutine(mock.Mock())) - on_request_end = mock.Mock(side_effect=asyncio.coroutine(mock.Mock())) + on_request_start = mock.Mock(side_effect=make_mocked_coro(mock.Mock())) + on_request_end = mock.Mock(side_effect=make_mocked_coro(mock.Mock())) on_dns_resolvehost_start = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock())) + side_effect=make_mocked_coro(mock.Mock())) on_dns_resolvehost_end = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock())) - on_request_redirect = mock.Mock(side_effect=asyncio.coroutine(mock.Mock())) + side_effect=make_mocked_coro(mock.Mock())) + on_request_redirect = mock.Mock(side_effect=make_mocked_coro(mock.Mock())) on_connection_create_start = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock())) + side_effect=make_mocked_coro(mock.Mock())) on_connection_create_end = mock.Mock( - side_effect=asyncio.coroutine(mock.Mock())) + side_effect=make_mocked_coro(mock.Mock())) async def redirector(request): raise web.HTTPFound(location=URL('/redirected')) diff --git a/tests/test_web_protocol.py b/tests/test_web_protocol.py index 3ea5ad950f0..90da07ebae2 100644 --- a/tests/test_web_protocol.py +++ b/tests/test_web_protocol.py @@ -42,9 +42,12 @@ def srv(make_srv, transport): srv = make_srv() srv.connection_made(transport) transport.close.side_effect = partial(srv.connection_lost, None) - srv._drain_helper = mock.Mock() - srv._drain_helper.side_effect = helpers.noop - return srv + with mock.patch.object( + web.RequestHandler, + '_drain_helper', + side_effect=helpers.noop + ): + yield srv @pytest.fixture @@ -400,19 +403,21 @@ async def test_lingering(srv, transport) -> None: async def handle(message, request, writer): pass - srv.handle_request = handle - srv.data_received( - b'GET / HTTP/1.0\r\n' - b'Host: example.com\r\n' - b'Content-Length: 3\r\n\r\n') + with mock.patch.object( + web.RequestHandler, 'handle_request', create=True, new=handle + ): + srv.data_received( + b'GET / HTTP/1.0\r\n' + b'Host: example.com\r\n' + b'Content-Length: 3\r\n\r\n') - await asyncio.sleep(0.05) - assert not transport.close.called + await asyncio.sleep(0.05) + assert not transport.close.called - srv.data_received(b'123') + srv.data_received(b'123') - await asyncio.sleep(0) - transport.close.assert_called_with() + await asyncio.sleep(0) + transport.close.assert_called_with() async def test_lingering_disabled(make_srv, @@ -491,18 +496,19 @@ async def test_handle_cancel(make_srv, transport) -> None: async def handle_request(message, payload, writer): await asyncio.sleep(10) - srv.handle_request = handle_request - async def cancel(): srv._task_handler.cancel() - srv.data_received( - b'GET / HTTP/1.0\r\n' - b'Content-Length: 10\r\n' - b'Host: example.com\r\n\r\n') + with mock.patch.object( + web.RequestHandler, 'handle_request', create=True, new=handle_request + ): + srv.data_received( + b'GET / HTTP/1.0\r\n' + b'Content-Length: 10\r\n' + b'Host: example.com\r\n\r\n') - await asyncio.gather(srv._task_handler, cancel()) - assert log.debug.called + await asyncio.gather(srv._task_handler, cancel()) + assert log.debug.called async def test_handle_cancelled(make_srv, transport) -> None: @@ -513,7 +519,6 @@ async def test_handle_cancelled(make_srv, transport) -> None: srv = make_srv(logger=log) srv.connection_made(transport) - srv.handle_request = mock.Mock() # start request_handler task await asyncio.sleep(0) @@ -535,29 +540,31 @@ async def test_handle_400(srv, buf, transport) -> None: async def test_keep_alive(make_srv, transport, ceil) -> None: loop = asyncio.get_event_loop() srv = make_srv(keepalive_timeout=0.05) - srv.KEEPALIVE_RESCHEDULE_DELAY = 0.1 - srv.connection_made(transport) - - srv.keep_alive(True) - srv.handle_request = mock.Mock() - srv.handle_request.return_value = loop.create_future() - srv.handle_request.return_value.set_result(1) - - srv.data_received( - b'GET / HTTP/1.1\r\n' - b'Host: example.com\r\n' - b'Content-Length: 0\r\n\r\n') - - waiter = None - while waiter is None: - await asyncio.sleep(0) - waiter = srv._waiter - assert srv._keepalive_handle is not None - assert not transport.close.called - - await asyncio.sleep(0.2) - assert transport.close.called - assert waiter.cancelled + future = loop.create_future() + future.set_result(1) + + with mock.patch.object( + web.RequestHandler, 'KEEPALIVE_RESCHEDULE_DELAY', new=0.1 + ), mock.patch.object( + web.RequestHandler, 'handle_request', create=True, return_value=future + ): + srv.connection_made(transport) + srv.keep_alive(True) + srv.data_received( + b'GET / HTTP/1.1\r\n' + b'Host: example.com\r\n' + b'Content-Length: 0\r\n\r\n') + + waiter = None + while waiter is None: + await asyncio.sleep(0) + waiter = srv._waiter + assert srv._keepalive_handle is not None + assert not transport.close.called + + await asyncio.sleep(0.2) + assert transport.close.called + assert waiter.cancelled async def test_srv_process_request_without_timeout(make_srv, @@ -634,29 +641,34 @@ async def test_close(srv, transport) -> None: srv.connection_made(transport) await asyncio.sleep(0) - srv.handle_request = mock.Mock() - srv.handle_request.side_effect = helpers.noop + handle_request = mock.Mock() + handle_request.side_effect = helpers.noop + with mock.patch.object( + web.RequestHandler, + 'handle_request', + create=True, + new=handle_request + ): + assert transport is srv.transport + + srv._keepalive = True + srv.data_received( + b'GET / HTTP/1.1\r\n' + b'Host: example.com\r\n' + b'Content-Length: 0\r\n\r\n' + b'GET / HTTP/1.1\r\n' + b'Host: example.com\r\n' + b'Content-Length: 0\r\n\r\n') - assert transport is srv.transport - - srv._keepalive = True - srv.data_received( - b'GET / HTTP/1.1\r\n' - b'Host: example.com\r\n' - b'Content-Length: 0\r\n\r\n' - b'GET / HTTP/1.1\r\n' - b'Host: example.com\r\n' - b'Content-Length: 0\r\n\r\n') - - await asyncio.sleep(0.05) - assert srv._task_handler - assert srv._waiter + await asyncio.sleep(0.05) + assert srv._task_handler + assert srv._waiter - srv.close() - await asyncio.sleep(0) - assert srv._task_handler is None - assert srv.transport is None - assert transport.close.called + srv.close() + await asyncio.sleep(0) + assert srv._task_handler is None + assert srv.transport is None + assert transport.close.called async def test_pipeline_multiple_messages( @@ -839,7 +851,10 @@ async def handler(request): app.router.add_route('POST', '/', handler) server = await aiohttp_server(app, logger=logger) - _, writer = await asyncio.open_connection('127.0.0.1', server.port) + if helpers.PY_38: + writer = await asyncio.connect('127.0.0.1', server.port) + else: + _, writer = await asyncio.open_connection('127.0.0.1', server.port) writer.write("""POST / HTTP/1.1\r Connection: keep-alive\r Content-Length: 10\r