Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Source Exchange Rates: add ignore_weekends boolean option #7936

Merged
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ RUN pip install .
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

LABEL io.airbyte.version=0.2.4
LABEL io.airbyte.version=0.2.5
LABEL io.airbyte.name=airbyte/source-exchange-rates
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,12 @@ class ExchangeRates(HttpStream):
cursor_field = date_field_name
primary_key = ""

def __init__(self, base: Optional[str], start_date: DateTime, access_key: str):
def __init__(self, base: Optional[str], start_date: DateTime, access_key: str, ignore_weekends: Optional[bool]):
super().__init__()
self._base = base
self._start_date = start_date
self.access_key = access_key
self.ignore_weekends = ignore_weekends

def path(
self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
Expand All @@ -52,7 +53,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]:
stream_state = stream_state or {}
start_date = pendulum.parse(stream_state.get(self.date_field_name, self._start_date))
return chunk_date_range(start_date)
return chunk_date_range(start_date, self.ignore_weekends)

def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]):
current_stream_state = current_stream_state or {}
Expand All @@ -62,7 +63,7 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late
return current_stream_state


def chunk_date_range(start_date: DateTime) -> Iterable[Mapping[str, Any]]:
def chunk_date_range(start_date: DateTime, ignore_weekends: bool) -> Iterable[Mapping[str, Any]]:
"""
Returns a list of each day between the start date and now. Ignore weekends since exchanges don't run on weekends.
The return value is a list of dicts {'date': date_string}.
Expand All @@ -71,7 +72,7 @@ def chunk_date_range(start_date: DateTime) -> Iterable[Mapping[str, Any]]:
now = pendulum.now()
while start_date < now:
day_of_week = start_date.day_of_week
if day_of_week != pendulum.SATURDAY and day_of_week != pendulum.SUNDAY:
if day_of_week != pendulum.SATURDAY and day_of_week != pendulum.SUNDAY or not ignore_weekends:
days.append({"date": start_date.to_date_string()})
start_date = start_date.add(days=1)

Expand Down Expand Up @@ -106,4 +107,4 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
return False, e

def streams(self, config: Mapping[str, Any]) -> List[Stream]:
return [ExchangeRates(config.get("base"), config["start_date"], config["access_key"])]
return [ExchangeRates(config.get("base"), config["start_date"], config["access_key"], config["ignore_weekends"])]
lizdeika marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@
"type": "string",
"description": "ISO reference currency. See <a href=\"https://www.ecb.europa.eu/stats/policy_and_exchange_rates/euro_reference_exchange_rates/html/index.en.html\">here</a>. Free plan doesn't support Source Currency Switching, default base currency is EUR",
"examples": ["EUR", "USD"]
},
"ignore_weekends": {
"type": "boolean",
"description": "Ignore weekends? (Exchanges don't run on weekends)",
"default": true
}
}
}
Expand Down