Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set default DHT num_workers = 4 #342

Merged
merged 10 commits into from
Jul 31, 2021
Merged

Set default DHT num_workers = 4 #342

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
20a4899
Set default DHTNode.num_workers = 8
borzunov Jul 28, 2021
2a02de0
Add environment variable for DHT max workers
dvmazur Jul 29, 2021
2e0671e
add dht_max_workers argument for dht benchmark and refactor dht
dvmazur Jul 29, 2021
af09245
fix suggestions
dvmazur Jul 30, 2021
1bc5e8f
run black on benchmarks
dvmazur Jul 30, 2021
c278a8a
Remove excess import
borzunov Jul 30, 2021
90a81c4
Set default to 4
borzunov Jul 30, 2021
7505202
Rename max_workers -> num_workers
borzunov Jul 30, 2021
ea52c9a
Merge branch 'master' into num-workers
borzunov Jul 31, 2021
10ad6e8
Fix bug in reading env variable
borzunov Jul 31, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions hivemind/dht/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

from multiaddr import Multiaddr

from hivemind.dht.node import DHTNode
from hivemind.dht.node import DEFAULT_NUM_WORKERS, DHTNode
from hivemind.dht.routing import DHTID, DHTKey, DHTValue, Subkey
from hivemind.dht.validation import CompositeValidator, RecordValidatorBase
from hivemind.p2p import P2P, PeerID
Expand Down Expand Up @@ -62,7 +62,7 @@ def __init__(
*,
start: bool,
daemon: bool = True,
max_workers: Optional[int] = None,
max_workers: int = DEFAULT_NUM_WORKERS,
record_validators: Iterable[RecordValidatorBase] = (),
shutdown_timeout: float = 3,
await_ready: bool = True,
Expand Down Expand Up @@ -106,7 +106,7 @@ def run(self) -> None:
async def _run():
self._node = await DHTNode.create(
initial_peers=self.initial_peers,
num_workers=self.max_workers or 1,
num_workers=self.max_workers,
record_validator=self._record_validator,
**self.kwargs,
)
Expand Down
8 changes: 6 additions & 2 deletions hivemind/dht/node.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import asyncio
import dataclasses
import os
import random
from collections import defaultdict, Counter
from dataclasses import dataclass, field
Expand Down Expand Up @@ -38,6 +39,9 @@
logger = get_logger(__name__)


DEFAULT_NUM_WORKERS = os.environ.get("HIVEMIND_DHT_NUM_WORKERS", 8)


class DHTNode:
"""
Asyncio-based class that represents one DHT participant. Created via await DHTNode.create(...)
Expand Down Expand Up @@ -110,7 +114,7 @@ async def create(
cache_refresh_before_expiry: float = 5,
cache_on_store: bool = True,
reuse_get_requests: bool = True,
num_workers: int = 1,
num_workers: int = DEFAULT_NUM_WORKERS,
chunk_size: int = 16,
blacklist_time: float = 5.0,
backoff_rate: float = 2.0,
Expand Down Expand Up @@ -154,7 +158,7 @@ async def create(
:param backoff_rate: blacklist time will be multiplied by :backoff_rate: for each successive non-response
:param validate: if True, use initial peers to validate that this node is accessible and synchronized
:param strict: if True, any error encountered in validation will interrupt the creation of DHTNode
:param client_mode: if False (default), this node will accept incoming requests as a full DHT "citzen"
:param client_mode: if False (default), this node will accept incoming requests as a full DHT "citizen"
if True, this node will refuse any incoming requests, effectively being only a client
:param record_validator: instance of RecordValidatorBase used for signing and validating stored records
:param authorizer: instance of AuthorizerBase used for signing and validating requests and response
Expand Down