✨ Source Facebook-Marketing: Add Selectable Auth (with migration & tests)

airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml

@@ -6,9 +6,10 @@ acceptance_tests:
   spec:
     tests:
       - spec_path: "integration_tests/spec.json"
+        # the source now supports 2 auth types, thus we should skip the check,
+        # this change is intentional
         backward_compatibility_tests_config:
-          disable_for_version: "1.2.2"
-          previous_connector_version: "1.2.1"
+          disable_for_version: "3.1.0"
   connection:
     tests:
       - config_path: "secrets/config.json"

airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json

@@ -26,6 +26,72 @@
         "airbyte_secret": true,
         "type": "string"
       },
+      "credentials": {
+        "title": "Authentication",
+        "description": "Credentials for connecting to the Facebook Marketing API",
+        "type": "object",
+        "discriminator": {
+          "propertyName": "auth_type",
+          "mapping": {
+            "Client": "#/definitions/OAuthCredentials",
+            "Service": "#/definitions/ServiceAccountCredentials"
+          }
+        },
+        "oneOf": [
+          {
+            "title": "Authenticate via Facebook Marketing (Oauth)",
+            "type": "object",
+            "properties": {
+              "auth_type": {
+                "title": "Auth Type",
+                "default": "Client",
+                "const": "Client",
+                "enum": ["Client"],
+                "type": "string"
+              },
+              "client_id": {
+                "title": "Client ID",
+                "description": "Client ID for the Facebook Marketing API",
+                "airbyte_secret": true,
+                "type": "string"
+              },
+              "client_secret": {
+                "title": "Client Secret",
+                "description": "Client Secret for the Facebook Marketing API",
+                "airbyte_secret": true,
+                "type": "string"
+              },
+              "access_token": {
+                "title": "Access Token",
+                "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions <b>ads_management, ads_read, read_insights, business_management</b>. Then click on \"Get token\". See the <a href=\"https://docs.airbyte.com/integrations/sources/facebook-marketing\">docs</a> for more information.",
+                "airbyte_secret": true,
+                "type": "string"
+              }
+            },
+            "required": ["client_id", "client_secret", "auth_type"]
+          },
+          {
+            "title": "Service Account Key Authentication",
+            "type": "object",
+            "properties": {
+              "auth_type": {
+                "title": "Auth Type",
+                "default": "Service",
+                "const": "Service",
+                "enum": ["Service"],
+                "type": "string"
+              },
+              "access_token": {
+                "title": "Access Token",
+                "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions <b>ads_management, ads_read, read_insights, business_management</b>. Then click on \"Get token\". See the <a href=\"https://docs.airbyte.com/integrations/sources/facebook-marketing\">docs</a> for more information.",
+                "airbyte_secret": true,
+                "type": "string"
+              }
+            },
+            "required": ["access_token", "auth_type"]
+          }
+        ]
+      },
       "start_date": {
         "title": "Start Date",
         "description": "The date from which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. If not set then all data will be replicated for usual streams and only last 2 years for insight streams.",
@@ -462,19 +528,21 @@
         "type": "string"
       }
     },
-    "required": ["account_ids", "access_token"]
+    "required": ["account_ids"]
   },
   "supportsIncremental": true,
   "supported_destination_sync_modes": ["append"],
   "advanced_auth": {
     "auth_flow_type": "oauth2.0",
+    "predicate_key": ["credentials", "auth_type"],
+    "predicate_value": "Client",
     "oauth_config_specification": {
       "complete_oauth_output_specification": {
         "type": "object",
         "properties": {
           "access_token": {
             "type": "string",
-            "path_in_connector_config": ["access_token"]
+            "path_in_connector_config": ["credentials", "access_token"]
           }
         }
       },
@@ -495,11 +563,11 @@
         "properties": {
           "client_id": {
             "type": "string",
-            "path_in_connector_config": ["client_id"]
+            "path_in_connector_config": ["credentials", "client_id"]
           },
           "client_secret": {
             "type": "string",
-            "path_in_connector_config": ["client_secret"]
+            "path_in_connector_config": ["credentials", "client_secret"]
           }
         }
       }

airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml

@@ -10,7 +10,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
-  dockerImageTag: 3.1.0
+  dockerImageTag: 3.2.0
   dockerRepository: airbyte/source-facebook-marketing
   documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing
   githubIssueLabel: source-facebook-marketing

airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "3.1.0"
+version = "3.2.0"
 name = "source-facebook-marketing"
 description = "Source implementation for Facebook Marketing."
 authors = [ "Airbyte <[email protected]>",]

airbyte-integrations/connectors/source-facebook-marketing/sample_files/sample_config.json

@@ -2,5 +2,8 @@
   "start_date": "2020-09-25T00:00:00Z",
   "end_date": "2021-01-01T00:00:00Z",
   "account_id": "<YOUR_ACCOUNT_ID_HERE>",
-  "access_token": "<YOUR_TOKEN_HERE>"
+  "credentials": {
+    "auth_type": "Service",
+    "access_token": "<YOUR_ACCESS_TOKEN_HERE>"
+  }
 }

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py

@@ -121,3 +121,73 @@ def transform(cls, config: Mapping[str, Any]) -> Mapping[str, Any]:
             config[stream_filter] = statuses
         # return transformed config
         return config
+
+
+class MigrateSecretsPathInConnector:
+    """
+    This class stands for migrating the config at runtime.
+    This migration is intended for backwards compatibility with the previous version, so existing secrets configurations gets migrated to new path.
+
+    Starting from `2.2.0`, the `client_id`, `client_secret` and `access_token` will be placed at `credentials` path.
+    """
+
+    @classmethod
+    def _should_migrate(cls, config: Mapping[str, Any]) -> bool:
+        """
+        This method determines whether the config should be migrated to nest existing fields at credentials.
+        It is assumed if credentials does not exist on configuration, `client_id`, `client_secret` and `access_token` exists on root path.
+        Returns:
+            > True, if the migration is necessary
+            > False, otherwise.
+        """
+        return "access_token" in config or "client_id" in config or "client_secret" in config
+
+    @classmethod
+    def migrate(cls, args: List[str], source: Source) -> None:
+        """
+        This method checks the input args, should the config be migrated,
+        transform if neccessary and emit the CONTROL message.
+        """
+        # get config path
+        config_path = AirbyteEntrypoint(source).extract_config(args)
+        # proceed only if `--config` arg is provided
+        if config_path:
+            # read the existing config
+            config = source.read_config(config_path)
+            # migration check
+            if cls._should_migrate(config):
+                cls._emit_control_message(
+                    cls._modify_and_save(config_path, source, config),
+                )
+
+    @classmethod
+    def _transform(cls, config: Mapping[str, Any]) -> Mapping[str, Any]:
+        # transform the config
+        if "credentials" not in config:
+            config["credentials"] = {
+                "auth_type": "Service",
+            }
+        if "access_token" in config:
+            config["credentials"]["access_token"] = config.pop("access_token")
+        if "client_id" in config:
+            config["credentials"]["auth_type"] = "Client"
+            config["credentials"]["client_id"] = config.pop("client_id")
+        if "client_secret" in config:
+            config["credentials"]["auth_type"] = "Client"
+            config["credentials"]["client_secret"] = config.pop("client_secret")
+        # return transformed config
+        return config
+
+    @classmethod
+    def _modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]:
+        # modify the config
+        migrated_config = cls._transform(config)
+        # save the config
+        source.write_config(migrated_config, config_path)
+        # return modified config
+        return migrated_config
+
+    @classmethod
+    def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None:
+        # add the Airbyte Control Message to message repo
+        print(create_connector_config_control_message(migrated_config).json(exclude_unset=True))

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/run.py

@@ -7,12 +7,13 @@
 
 from airbyte_cdk.entrypoint import launch
 
-from .config_migrations import MigrateAccountIdToArray, MigrateIncludeDeletedToStatusFilters
+from .config_migrations import MigrateAccountIdToArray, MigrateIncludeDeletedToStatusFilters, MigrateSecretsPathInConnector
 from .source import SourceFacebookMarketing
 
 
 def run():
     source = SourceFacebookMarketing()
     MigrateAccountIdToArray.migrate(sys.argv[1:], source)
     MigrateIncludeDeletedToStatusFilters.migrate(sys.argv[1:], source)
+    MigrateSecretsPathInConnector.migrate(sys.argv[1:], source)
     launch(source, sys.argv[1:])

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py

@@ -95,7 +95,10 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) ->
             if config.start_date and config.end_date < config.start_date:
                 return False, "End date must be equal or after start date."
 
-            api = API(access_token=config.access_token, page_size=config.page_size)
+            if config.credentials is not None:
+                api = API(access_token=config.credentials.access_token, page_size=config.page_size)
+            else:
+                api = API(access_token=config.access_token, page_size=config.page_size)
 
             for account_id in config.account_ids:
                 # Get Ad Account to check creds
@@ -129,7 +132,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]:
             config.start_date = validate_start_date(config.start_date)
             config.end_date = validate_end_date(config.start_date, config.end_date)
 
-        api = API(access_token=config.access_token, page_size=config.page_size)
+        if config.credentials is not None:
+            api = API(access_token=config.credentials.access_token, page_size=config.page_size)
+        else:
+            api = API(access_token=config.access_token, page_size=config.page_size)
 
         # if start_date not specified then set default start_date for report streams to 2 years ago
         report_start_date = config.start_date or pendulum.now().add(years=-2)
@@ -242,14 +248,16 @@ def spec(self, *args, **kwargs) -> ConnectorSpecification:
             connectionSpecification=ConnectorConfig.schema(),
             advanced_auth=AdvancedAuth(
                 auth_flow_type=AuthFlowType.oauth2_0,
+                predicate_key=["credentials", "auth_type"],
+                predicate_value="Client",
                 oauth_config_specification=OAuthConfigSpecification(
                     complete_oauth_output_specification={
                         "type": "object",
                         "properties": {
                             "access_token": {
                                 "type": "string",
-                                "path_in_connector_config": ["access_token"],
-                            }
+                                "path_in_connector_config": ["credentials", "access_token"],
+                            },
                         },
                     },
                     complete_oauth_server_input_specification={
@@ -265,11 +273,11 @@ def spec(self, *args, **kwargs) -> ConnectorSpecification:
                         "properties": {
                             "client_id": {
                                 "type": "string",
-                                "path_in_connector_config": ["client_id"],
+                                "path_in_connector_config": ["credentials", "client_id"],
                             },
                             "client_secret": {
                                 "type": "string",
-                                "path_in_connector_config": ["client_secret"],
+                                "path_in_connector_config": ["credentials", "client_secret"],
                             },
                         },
                     },

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py

@@ -4,9 +4,10 @@
 import logging
 from datetime import datetime, timezone
 from enum import Enum
-from typing import List, Optional, Set
+from typing import List, Literal, Optional, Set, Union
 
 from airbyte_cdk.sources.config import BaseConfig
+from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
 from facebook_business.adobjects.ad import Ad
 from facebook_business.adobjects.adset import AdSet
 from facebook_business.adobjects.adsinsights import AdsInsights
@@ -31,6 +32,48 @@
 EMPTY_PATTERN = "^$"
 
 
+class OAuthCredentials(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Authenticate via Facebook Marketing (Oauth)"
+        discriminator = "auth_type"
+
+    auth_type: Literal["Client"] = Field("Client", const=True)
+    client_id: str = Field(
+        title="Client ID",
+        description="Client ID for the Facebook Marketing API",
+        airbyte_secret=True,
+    )
+    client_secret: str = Field(
+        title="Client Secret",
+        description="Client Secret for the Facebook Marketing API",
+        airbyte_secret=True,
+    )
+    access_token: Optional[str] = Field(
+        title="Access Token",
+        description="The value of the generated access token. "
+        'From your App’s Dashboard, click on "Marketing API" then "Tools". '
+        'Select permissions <b>ads_management, ads_read, read_insights, business_management</b>. Then click on "Get token". '
+        'See the <a href="https://docs.airbyte.com/integrations/sources/facebook-marketing">docs</a> for more information.',
+        airbyte_secret=True,
+    )
+
+
+class ServiceAccountCredentials(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Service Account Key Authentication"
+        discriminator = "auth_type"
+
+    auth_type: Literal["Service"] = Field("Service", const=True)
+    access_token: str = Field(
+        title="Access Token",
+        description="The value of the generated access token. "
+        'From your App’s Dashboard, click on "Marketing API" then "Tools". '
+        'Select permissions <b>ads_management, ads_read, read_insights, business_management</b>. Then click on "Get token". '
+        'See the <a href="https://docs.airbyte.com/integrations/sources/facebook-marketing">docs</a> for more information.',
+        airbyte_secret=True,
+    )
+
+
 class InsightConfig(BaseModel):
     """Config for custom insights"""
 
@@ -142,7 +185,7 @@ class Config:
         min_items=1,
     )
 
-    access_token: str = Field(
+    access_token: Optional[str] = Field(
         title="Access Token",
         order=1,
         description=(
@@ -154,6 +197,13 @@ class Config:
         airbyte_secret=True,
     )
 
+    credentials: Optional[Union[OAuthCredentials, ServiceAccountCredentials]] = Field(
+        title="Authentication",
+        description="Credentials for connecting to the Facebook Marketing API",
+        discriminator="auth_type",
+        type="object",
+    )
+
     start_date: Optional[datetime] = Field(
         title="Start Date",
         order=2,

airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py

@@ -26,6 +26,10 @@ def __init__(self) -> None:
         self._config: MutableMapping[str, Any] = {
             "account_ids": [ACCOUNT_ID],
             "access_token": ACCESS_TOKEN,
+            "credentials": {
+                "auth_type": "Service",
+                "access_token": ACCESS_TOKEN,
+            },
             "start_date": START_DATE,
             "end_date": END_DATE,
             "include_deleted": True,