From 6f925de4d6f0b6e41472c615757bbabd5bf12cb6 Mon Sep 17 00:00:00 2001 From: Chris Wu Date: Fri, 17 Sep 2021 10:40:53 -0700 Subject: [PATCH 1/2] Error handling for streams that are not applicable to some repos --- .../source-github/source_github/streams.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index 7a8c50e1550c..8a4c95059e0d 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -143,7 +143,18 @@ def read_records(self, stream_slice: Mapping[str, any] = None, **kwargs) -> Iter elif e.response.status_code == requests.codes.NOT_FOUND and "/teams?" in error_msg: # For private repositories `Teams` stream is not available and we get "404 Client Error: Not Found for # url: https://api.github.com/orgs/sherifnada/teams?per_page=100" error. - error_msg = f"Syncing `Team` stream isn't available for repository `{stream_slice['repository']}`." + error_msg = f"Syncing `Team` stream isn't available for organization `{stream_slice['organization']}`." + elif e.response.status_code == requests.codes.GONE and "/projects?" in error_msg: + # Some repos don't have projects enabled and we we get "410 Client Error: Gone for + # url: https://api.github.com/repos/xyz/projects?per_page=100" error. + error_msg = f"Syncing `Projects` stream isn't available for repository `{stream_slice['repository']}`." + elif e.response.status_code == requests.codes.NOT_FOUND and "/orgs/" in error_msg: + # Some streams are not available for repositories owned by a user instead of an organization. + # Handle "404 Client Error: Not Found" errors + if isinstance(self, Repositories): + error_msg = f"Syncing `Repositories` stream isn't available for repository `{stream_slice['organization']}`." + elif isinstance(self, Organizations): + error_msg = f"Syncing `Organizations` stream isn't available for organization `{stream_slice['organization']}`." elif e.response.status_code == requests.codes.CONFLICT: error_msg = ( f"Syncing `{self.name}` stream isn't available for repository " From 8974668d6cb5f5f947343745eeb64fc119ae7f43 Mon Sep 17 00:00:00 2001 From: Chris Wu Date: Fri, 17 Sep 2021 10:49:35 -0700 Subject: [PATCH 2/2] Handle users stream error --- .../connectors/source-github/source_github/streams.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index 8a4c95059e0d..18f5a68505ec 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -152,9 +152,14 @@ def read_records(self, stream_slice: Mapping[str, any] = None, **kwargs) -> Iter # Some streams are not available for repositories owned by a user instead of an organization. # Handle "404 Client Error: Not Found" errors if isinstance(self, Repositories): - error_msg = f"Syncing `Repositories` stream isn't available for repository `{stream_slice['organization']}`." + error_msg = f"Syncing `Repositories` stream isn't available for organization `{stream_slice['organization']}`." + elif isinstance(self, Users): + error_msg = f"Syncing `Users` stream isn't available for organization `{stream_slice['organization']}`." elif isinstance(self, Organizations): error_msg = f"Syncing `Organizations` stream isn't available for organization `{stream_slice['organization']}`." + else: + self.logger.error(f"Undefined error while reading records: {error_msg}") + raise e elif e.response.status_code == requests.codes.CONFLICT: error_msg = ( f"Syncing `{self.name}` stream isn't available for repository "