Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🎉 Source Facebook: rate limit not always handled #3525 #3600

Merged
merged 13 commits into from
May 26, 2021
Prev Previous commit
Next Next commit
fix tests
  • Loading branch information
eugene-kulak committed May 25, 2021
commit e21a25b130b50bf18e0df8d3011ee96f92eb88cb
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@

TEST_REQUIREMENTS = [
"pytest~=6.1",
"pytest-mock~=3.6",
"requests_mock~=1.8",
]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@

FACEBOOK_UNKNOWN_ERROR_CODE = 99
FACEBOOK_API_CALL_LIMIT_ERROR_CODES = (4, 17, 32, 613, 8000, 80004, 80003, 80002, 80005, 80006, 80001, 80008)
DEFAULT_SLEEP_INTERVAL = pendulum.Interval(minutes=1)


class FacebookAPIException(Exception):
Expand All @@ -50,7 +51,7 @@ def batch(iterable: Sequence, size: int = 1):


def handle_call_rate_response(exc: FacebookRequestError) -> bool:
pause_time = pendulum.Interval(minutes=1)
pause_time = DEFAULT_SLEEP_INTERVAL
platform_header = exc.http_headers().get("x-app-usage") or exc.http_headers().get("x-ad-account-usage")
if platform_header:
platform_header = json.loads(platform_header)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,23 @@

import json

import pendulum
import pytest
from airbyte_cdk.models import AirbyteStream
from facebook_business import FacebookSession
from source_facebook_marketing.client import Client


@pytest.fixture(name="some_config")
@pytest.fixture(scope="session", name="some_config")
def some_config_fixture():
return {"start_date": "2021-01-23T00:00:00Z", "account_id": "unknown_account", "access_token": "unknown_token"}


@pytest.fixture(autouse=True)
def mock_default_sleep_interval(mocker):
mocker.patch("source_facebook_marketing.client.common.DEFAULT_SLEEP_INTERVAL", return_value=pendulum.Interval(seconds=5))


@pytest.fixture(name="client")
def client_fixture(some_config):
client = Client(**some_config)
Expand Down Expand Up @@ -80,13 +86,15 @@ def fb_account_response_fixture():
class TestBackoff:
def test_limit_reached(self, requests_mock, client, fb_call_rate_response, fb_account_response):
"""Error once, check that we retry and not fail"""
responses = [
campaign_responses = [
fb_call_rate_response,
{"json": {"data": [1, 2]}, "status_code": 200},
{"json": {"data": [{"id": 1, "updated_time": "2020-09-25T00:00:00Z"}, {"id": 2, "updated_time": "2020-09-25T00:00:00Z"}]}, "status_code": 200},
]

requests_mock.register_uri("GET", FacebookSession.GRAPH + "/v10.0/me/adaccounts", [fb_account_response])
requests_mock.register_uri("GET", FacebookSession.GRAPH + "/v10.0/act_unknown_account/campaigns", responses)
requests_mock.register_uri("GET", FacebookSession.GRAPH + "/v10.0/act_unknown_account/campaigns", campaign_responses)
requests_mock.register_uri("GET", FacebookSession.GRAPH + "/v10.0/1/", [{"status_code": 200}])
requests_mock.register_uri("GET", FacebookSession.GRAPH + "/v10.0/2/", [{"status_code": 200}])

records = list(client.read_stream(AirbyteStream(name="campaigns", json_schema={})))

Expand All @@ -105,12 +113,14 @@ def test_batch_limit_reached(self, requests_mock, client, fb_call_rate_response)

assert not records

def test_server_error(self, requests_mock, client):
def test_server_error(self, requests_mock, client, fb_account_response):
"""Error once, check that we retry and not fail"""
responses = [
{"json": {"error": "something bad"}, "status_code": 500},
{"json": [], "status_code": 200},
]
requests_mock.register_uri("GET", FacebookSession.GRAPH, responses)

requests_mock.register_uri("GET", FacebookSession.GRAPH + "/v10.0/me/adaccounts", [fb_account_response])
requests_mock.register_uri("GET", FacebookSession.GRAPH + "/v10.0/act_unknown_account/campaigns", responses)

list(client.read_stream(AirbyteStream(name="campaigns", json_schema={})))