🎉 Instagram Source: Migrate to use CDK, upgrade API version and fix failures with HTTP status 400

airbyte-integrations/connectors/source-instagram/source_instagram/client/api.py

@@ -330,28 +330,21 @@ def list(self, fields: Sequence[str] = None) -> Iterator[dict]:
             ig_account = account["instagram_business_account"]
             media = self._get_media(ig_account, {"limit": self.result_return_limit}, ["media_type"])
             for ig_media in media:
-                try:
-                    yield {
-                        **{
-                            "id": ig_media.get("id"),
-                            "page_id": account["page_id"],
-                            "business_account_id": ig_account.get("id"),
-                        },
-                        **{record.get("name"): record.get("values")[0]["value"] for record in self._get_insights(ig_media)},
-                    }
-                except FacebookRequestError as error:
-                    # An error might occur if the media was posted before the most recent time that
-                    # the user's account was converted to a business account from a personal account
-                    if error.api_error_subcode() == 2108006:
-                        logger.error(f"Insights error for business_account_id {ig_account.get('id')}: {error.body()}")
-
-                        # We receive all Media starting from the last one, and if on the next Media we get an Insight error,
-                        # then no reason to make inquiries for each Media further, since they were published even earlier.
-                        break
-                    raise error
+                account_id = ig_account.get("id")
+                media_insights = self._get_insights(ig_media, account_id)
+                if media_insights is None:
+                    break
+                yield {
+                    **{
+                        "id": ig_media.get("id"),
+                        "page_id": account["page_id"],
+                        "business_account_id": account_id,
+                    },
+                    **{record.get("name"): record.get("values")[0]["value"] for record in media_insights},
+                }
 
     @backoff_policy
-    def _get_insights(self, item) -> Iterator[Any]:
+    def _get_insights(self, item, account_id) -> Optional[Iterator[Any]]:
         """
         This is necessary because the functions that call this endpoint return
         a generator, whose calls need decorated with a backoff.
@@ -363,7 +356,18 @@ def _get_insights(self, item) -> Iterator[Any]:
         else:
             metrics = self.MEDIA_METRICS
 
-        return item.get_insights(params={"metric": metrics})
+        try:
+            return item.get_insights(params={"metric": metrics})
+        except FacebookRequestError as error:
+            # An error might occur if the media was posted before the most recent time that
+            # the user's account was converted to a business account from a personal account
+            if error.api_error_subcode() == 2108006:
+                logger.error(f"Insights error for business_account_id {account_id}: {error.body()}")
+
+                # We receive all Media starting from the last one, and if on the next Media we get an Insight error,
+                # then no reason to make inquiries for each Media further, since they were published even earlier.
+                return None
+            raise error
 
 
 class StoriesInsightsAPI(StoriesAPI):

airbyte-integrations/connectors/source-instagram/source_instagram/client/client.py

@@ -40,7 +40,7 @@
 from .api import MediaAPI, MediaInsightsAPI, StoriesAPI, StoriesInsightsAPI, UserInsightsAPI, UserLifetimeInsightsAPI, UsersAPI
 from .common import InstagramAPIException, retry_pattern
 
-backoff_policy = retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5)
+backoff_policy = retry_pattern(backoff.expo, FacebookRequestError, max_tries=4, factor=5)
 
 
 class Client(BaseClient):

airbyte-integrations/connectors/source-instagram/source_instagram/client/common.py

@@ -44,6 +44,8 @@ def should_retry_api_error(exc):
         if (
             exc.http_status() == status_codes.TOO_MANY_REQUESTS
             or (exc.http_status() == status_codes.FORBIDDEN and exc.api_error_message() == "(#4) Application request limit reached")
+            or exc.http_status()
+            == status_codes.BAD_REQUEST  # Issue 4028, Sometimes an error about the Rate Limit is returned with a 400 HTTP code
             or exc.api_transient_error()
         ):
             return True