Replace chardet with hardcoded "utf8" encoding on saving and reading …

…files

Ref: #15

linguee_api/downloaders/file_cache.py

@@ -2,7 +2,6 @@
 import urllib.parse
 
 from linguee_api.downloaders.interfaces import IDownloader
-from linguee_api.utils import read_text
 
 
 class FileCache(IDownloader):
@@ -17,8 +16,8 @@ async def download(self, url: str) -> str:
         page_file = self.get_page_file(url)
         if not page_file.is_file():
             page = await self.upstream.download(url)
-            page_file.write_text(page)
-        return read_text(page_file)
+            page_file.write_text(page, encoding="utf-8")
+        return page_file.read_text(encoding="utf-8")
 
     def get_page_file(self, url: str) -> pathlib.Path:
         return self.cache_directory / urllib.parse.quote(url, safe="")

linguee_api/utils.py

@@ -1,14 +1,4 @@
 import importlib
-import pathlib
-
-import chardet
-
-
-def read_text(path: pathlib.Path) -> str:
-    """Read text from a file."""
-    content = path.read_bytes()
-    encoding = chardet.detect(content)["encoding"]
-    return content.decode(encoding)
 
 
 def import_string(import_name: str):

pyproject.toml

@@ -20,7 +20,6 @@ fastapi = "^0.63.0"
 pydantic = "^1.8.1"
 xextract = "^0.1.8"
 httpx = "^0.17.1"
-chardet = "^4.0.0"
 uvicorn = "^0.13.4"
 sentry-sdk = "^1.0.0"
 python-dotenv = "^0.17.0"

tests/parsers/test_search_result.py

@@ -90,6 +90,8 @@ async def test_parser_should_find_correction(
         ("einfach", "de", "en"),
         ("Tisch", "de", "en"),
         ("wünschen", "de", "en"),
+        ("envisage", "en", "zh"),
+        ("envisage", "en", "sv"),
     ],
 )
 @pytest.mark.asyncio