Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: Split WebSocket support into separate sync/async implementations #449

Merged
merged 24 commits into from
Dec 29, 2024
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 19 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -200,28 +200,38 @@ async with AsyncSession() as s:
### WebSockets

```python
from curl_cffi.requests import Session, WebSocket
from curl_cffi.requests import WebSocket

def on_message(ws: WebSocket, message):
def on_message(ws: WebSocket, message: str | bytes):
print(message)

with Session() as s:
ws = s.ws_connect(
"wss://api.gemini.com/v1/marketdata/BTCUSD",
on_message=on_message,
)
ws.run_forever()
ws = WebSocket(on_message=on_message)
ws.run_forever("wss://api.gemini.com/v1/marketdata/BTCUSD")
```

For low-level APIs, Scrapy integration and other advanced topics, see the
[docs](https://curl-cffi.readthedocs.io) for more details.

### asyncio WebSockets

```python
import asyncio
from curl_cffi.requests import AsyncSession

async with AsyncSession() as s:
ws = await s.ws_connect("wss://echo.websocket.org")
await asyncio.gather(*[ws.send_str("Hello, World!") for _ in range(10)])
async for message in ws:
print(message)
```

## Acknowledgement

- Originally forked from [multippt/python_curl_cffi](https://github.com/multippt/python_curl_cffi), which is under the MIT license.
- Headers/Cookies files are copied from [httpx](https://github.com/encode/httpx/blob/master/httpx/_models.py), which is under the BSD license.
- Asyncio support is inspired by Tornado's curl http client.
- The WebSocket API is inspired by [websocket_client](https://github.com/websocket-client/websocket-client).
- The synchronous WebSocket API is inspired by [websocket_client](https://github.com/websocket-client/websocket-client).
- The asynchronous WebSocket API is inspired by [aiohttp](https://github.com/aio-libs/aiohttp).


## Sponsor
Expand Down
23 changes: 15 additions & 8 deletions curl_cffi/curl.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
from __future__ import annotations

import re
import warnings
from http.cookies import SimpleCookie
from pathlib import Path
from typing import Any, List, Literal, Optional, Tuple, Union, cast
from typing import TYPE_CHECKING, Any, List, Literal, Optional, Tuple, Union, cast

import certifi

Expand All @@ -13,6 +15,15 @@
REASON_PHRASE_RE = re.compile(rb"HTTP/\d\.\d [0-9]{3} (.*)")
STATUS_LINE_RE = re.compile(rb"HTTP/(\d\.\d) ([0-9]{3}) (.*)")

if TYPE_CHECKING:

class CurlWsFrame:
age: int
flags: int
offset: int
bytesleft: int
len: int


class CurlError(Exception):
"""Base exception for curl_cffi package"""
Expand Down Expand Up @@ -234,7 +245,7 @@ def getinfo(self, option: CurlInfo) -> Union[bytes, int, float, List]:
0x200000: "long*",
0x300000: "double*",
0x400000: "struct curl_slist **",
0x500000: "long*"
0x500000: "long*",
}
ret_cast_option = {
0x100000: ffi.string,
Expand Down Expand Up @@ -310,7 +321,7 @@ def clean_after_perform(self, clear_headers: bool = True) -> None:
lib.curl_slist_free_all(self._proxy_headers)
self._proxy_headers = ffi.NULL

def duphandle(self) -> "Curl":
def duphandle(self) -> Curl:
lexiforest marked this conversation as resolved.
Show resolved Hide resolved
"""Wrapper for ``curl_easy_duphandle``.

This is not a full copy of entire curl object in python. For example, headers
Expand Down Expand Up @@ -379,7 +390,7 @@ def close(self) -> None:
ffi.release(self._error_buffer)
self._resolve = ffi.NULL

def ws_recv(self, n: int = 1024) -> Tuple[bytes, Any]:
def ws_recv(self, n: int = 1024) -> Tuple[bytes, CurlWsFrame]:
"""Receive a frame from a websocket connection.

Args:
Expand Down Expand Up @@ -422,10 +433,6 @@ def ws_send(self, payload: bytes, flags: CurlWsFlag = CurlWsFlag.BINARY) -> int:
self._check_error(ret, "WS_SEND")
return n_sent[0]

def ws_close(self) -> None:
dolfies marked this conversation as resolved.
Show resolved Hide resolved
"""Send the close frame."""
self.ws_send(b"", CurlWsFlag.CLOSE)


class CurlMime:
"""Wrapper for the ``curl_mime_`` API."""
Expand Down
3 changes: 2 additions & 1 deletion curl_cffi/requests/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
"Headers",
"Request",
"Response",
"AsyncWebSocket",
"WebSocket",
"WebSocketError",
"WsCloseCode",
Expand All @@ -38,7 +39,7 @@
from .impersonate import BrowserType, BrowserTypeLiteral, ExtraFingerprints, ExtraFpDict
from .models import Request, Response
from .session import AsyncSession, HttpMethod, ProxySpec, Session, ThreadType
from .websockets import WebSocket, WebSocketError, WsCloseCode
from .websockets import AsyncWebSocket, WebSocket, WebSocketError, WsCloseCode


def request(
Expand Down
4 changes: 3 additions & 1 deletion curl_cffi/requests/cookies.py
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,9 @@ def get_dict(self, domain: Optional[str] = None, path: Optional[str] = None) ->
"""
ret = {}
for cookie in self.jar:
if (domain is None or cookie.domain == domain) and (path is None or cookie.path == path):
if (domain is None or cookie.domain == domain) and (
path is None or cookie.path == path
):
ret[cookie.name] = cookie.value
return ret

Expand Down
Loading
Loading