add WebClient tests

refactor WebClient
remove log module

aioscrapy/client.py

@@ -63,15 +63,16 @@ async def fetch(self, key: str) -> Optional[VT]:
             return new_value
 
 
-class WebClient(Client[str, Tuple[ClientResponse, bytes]]):
+class WebClient(Client[str, ClientResponse]):
     def __init__(self, session_pool: SessionPool):
         self._session_pool = session_pool
 
-    async def fetch(self, key: str) -> Optional[Tuple[ClientResponse, bytes]]:
+    async def fetch(self, key: str) -> Optional[ClientResponse]:
         proxy, session = self._session_pool.rand()
         try:
             response: aiohttp.ClientResponse = await session.get(key, proxy=proxy)
-            return response, await response.read()
+            await response.read()
+            return response
         except (aiohttp.ClientHttpProxyError, aiohttp.ClientProxyConnectionError):
             if proxy is not None:
                 self._session_pool.pop(proxy)
@@ -87,11 +88,10 @@ def __init__(self, session_pool: SessionPool):
 
     async def fetch(self, key: str) -> Optional[str]:
         client = WebClient(self._session_pool)
-        response_and_body = await client.fetch(key)
-        if response_and_body is None:
+        response = await client.fetch(key)
+        if response is None:
             return None
-        response, body = response_and_body
-        return body.decode(response.get_encoding())
+        return await response.text()
 
 
 class WebByteClient(Client[str, bytes]):
@@ -100,11 +100,11 @@ def __init__(self, session_pool: SessionPool):
 
     async def fetch(self, key: str) -> Optional[bytes]:
         client = WebClient(self._session_pool)
-        response_and_body = await client.fetch(key)
-        if response_and_body is None:
+        response = await client.fetch(key)
+        if response is None:
             return None
-        response, body = response_and_body
-        return body
+        # noinspection PyProtectedMember
+        return response._body
 
 
 class RetryClient(Client[KT, VT]):

setup.py

@@ -11,7 +11,7 @@
 EMAIL = "[email protected]"
 AUTHOR = "eugen1j"
 REQUIRES_PYTHON = ">=3.7.0"
-VERSION = "0.1.5"
+VERSION = "0.1.6"
 
 here = os.path.abspath(os.path.dirname(__file__))
 with open(f"{here}/README.md") as f:

tests/test_client.py

@@ -1,9 +1,13 @@
 from typing import Optional
 
 import pytest
+from aiohttp import ClientResponse
+
+from aioscrapy import SingleSessionPool
 
 from aioscrapy.cache import FakeCache
-from aioscrapy.client import Client, FakeClient, CacheClient, RetryClient, CacheOnlyClient, CacheSkipClient
+from aioscrapy.client import Client, FakeClient, CacheClient, RetryClient, CacheOnlyClient, CacheSkipClient, WebClient, \
+    WebTextClient, WebByteClient
 
 
 class ForRetryClient(Client[str, str]):
@@ -94,3 +98,24 @@ async def test_retry_client_enough_tries():
     )
     key = 'key'
     assert await client.fetch(key) is key
+
+
+@pytest.mark.asyncio
+async def test_web_client_fetch_google():
+    client = WebClient(SingleSessionPool())
+    response = await client.fetch('https://google.com')
+    assert isinstance(response, ClientResponse)
+
+
+@pytest.mark.asyncio
+async def test_web_text_client_fetch_google():
+    client = WebTextClient(SingleSessionPool())
+    response = await client.fetch('https://google.com')
+    assert isinstance(response, str)
+
+
+@pytest.mark.asyncio
+async def test_web_byte_client_fetch_google():
+    client = WebByteClient(SingleSessionPool())
+    response = await client.fetch('https://google.com')
+    assert isinstance(response, bytes)