Skip to content

Commit

Permalink
Merge pull request #118 from Breakthrough-Energy/jon/requests2
Browse files Browse the repository at this point in the history
Reduce throughput to nrel api by combining duplicate requests
  • Loading branch information
jenhagg authored Oct 16, 2020
2 parents c999c9e + 4d059be commit ce5535d
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 259 deletions.
10 changes: 9 additions & 1 deletion prereise/gather/request_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,12 @@
from urllib.error import HTTPError


class TransientError(Exception):
"""Used for errors which can be retried"""

pass


class RateLimit:
"""Provides a way to call an arbitrary function at most once per interval.
Expand Down Expand Up @@ -56,10 +62,12 @@ def retry(_func=None, retry_count=5, interval=None, allowed_exceptions=(HTTPErro

def decorator(func):
limiter = RateLimit(interval)
func.retry_count = 0

@functools.wraps(func)
def wrapper(*args, **kwargs):
for _ in range(retry_count):
for i in range(retry_count):
func.retry_count = i + 1
try:
return limiter.invoke(lambda: func(*args, **kwargs))
except allowed_exceptions as e:
Expand Down
Loading

0 comments on commit ce5535d

Please sign in to comment.