From eccc2a1211d8433f349f5e8b8c716c7ec8dc3b28 Mon Sep 17 00:00:00 2001 From: JohnMikkelsonOvative Date: Wed, 24 Apr 2024 16:21:36 -0500 Subject: [PATCH 01/20] EMRGE-1446 Selectable Auth working on local CLI. Still need to add changes to solve build issues, and changes to solve unit testing issues. --- .../source_facebook_marketing/source.py | 18 +++-- .../source_facebook_marketing/spec.py | 65 ++++++++++++------- 2 files changed, 55 insertions(+), 28 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 0e4744be48f29..1dd0051b1efe3 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -95,7 +95,10 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> if config.start_date and config.end_date < config.start_date: return False, "End date must be equal or after start date." - api = API(access_token=config.access_token, page_size=config.page_size) + if config.credentials.auth_type == 'Client': + api = API(access_token=config.credentials.refresh_token, page_size=config.page_size) + if config.credentials.auth_type == 'Service': + api = API(access_token=config.credentials.service_account_info, page_size=config.page_size) for account_id in config.account_ids: # Get Ad Account to check creds @@ -129,7 +132,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: config.start_date = validate_start_date(config.start_date) config.end_date = validate_end_date(config.start_date, config.end_date) - api = API(access_token=config.access_token, page_size=config.page_size) + if config.credentials.auth_type == 'Client': + api = API(access_token=config.credentials.refresh_token, page_size=config.page_size) + if config.credentials.auth_type == 'Service': + api = API(access_token=config.credentials.service_account_info, page_size=config.page_size) # if start_date not specified then set default start_date for report streams to 2 years ago report_start_date = config.start_date or pendulum.now().add(years=-2) @@ -242,13 +248,15 @@ def spec(self, *args, **kwargs) -> ConnectorSpecification: connectionSpecification=ConnectorConfig.schema(), advanced_auth=AdvancedAuth( auth_flow_type=AuthFlowType.oauth2_0, + predicate_key=["credentials", "auth_type"], + predicate_value="Client", oauth_config_specification=OAuthConfigSpecification( complete_oauth_output_specification={ "type": "object", "properties": { "access_token": { "type": "string", - "path_in_connector_config": ["access_token"], + "path_in_connector_config": ["credentials", "access_token"], } }, }, @@ -265,11 +273,11 @@ def spec(self, *args, **kwargs) -> ConnectorSpecification: "properties": { "client_id": { "type": "string", - "path_in_connector_config": ["client_id"], + "path_in_connector_config": ["credentials", "client_id"], }, "client_secret": { "type": "string", - "path_in_connector_config": ["client_secret"], + "path_in_connector_config": ["credentials", "client_secret"], }, }, }, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index 4b1b7a8a51e47..6a53d9c11002d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -5,9 +5,10 @@ import logging from datetime import datetime, timezone from enum import Enum -from typing import List, Optional, Set +from typing import List, Optional, Set, Literal, Union from airbyte_cdk.sources.config import BaseConfig +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig from facebook_business.adobjects.ad import Ad from facebook_business.adobjects.adset import AdSet from facebook_business.adobjects.adsinsights import AdsInsights @@ -27,6 +28,41 @@ EMPTY_PATTERN = "^$" +class OAuthCredentials(BaseModel): + class Config(OneOfOptionConfig): + title = "Authenticate via Facebook Marketing (Oauth)" + discriminator = "auth_type" + + auth_type: Literal["Client"] = Field("Client", const=True) + client_id: str = Field( + title="Client ID", + description="Client ID for the Facebook Marketing API", + airbyte_secret=True, + ) + client_secret: str = Field( + title="Client Secret", + description="Client Secret for the Google Drive API", + airbyte_secret=True, + ) + refresh_token: str = Field( + title="Refresh Token", + description="Refresh Token for the Google Drive API", + airbyte_secret=True, + ) + + +class ServiceAccountCredentials(BaseModel): + class Config(OneOfOptionConfig): + title = "Service Account Key Authentication" + discriminator = "auth_type" + + auth_type: Literal["Service"] = Field("Service", const=True) + service_account_info: str = Field( + title="Service Account Information", + description='The JSON key of the service account to use for authorization.', + airbyte_secret=True, + ) + class InsightConfig(BaseModel): """Config for custom insights""" @@ -138,16 +174,11 @@ class Config: min_items=1, ) - access_token: str = Field( - title="Access Token", - order=1, - description=( - "The value of the generated access token. " - 'From your App’s Dashboard, click on "Marketing API" then "Tools". ' - 'Select permissions ads_management, ads_read, read_insights, business_management. Then click on "Get token". ' - 'See the docs for more information.' - ), - airbyte_secret=True, + credentials: Union[OAuthCredentials, ServiceAccountCredentials] = Field( + title="Authentication", + description="Credentials for connecting to the Facebook Marketing API", + discriminator="auth_type", + type="object" ) start_date: Optional[datetime] = Field( @@ -254,15 +285,3 @@ class Config: default=True, airbyte_hidden=True, ) - - client_id: Optional[str] = Field( - description="The Client Id for your OAuth app", - airbyte_secret=True, - airbyte_hidden=True, - ) - - client_secret: Optional[str] = Field( - description="The Client Secret for your OAuth app", - airbyte_secret=True, - airbyte_hidden=True, - ) From cda978e05f4c7bfeb7e06a232f16ff0e24d2ed5a Mon Sep 17 00:00:00 2001 From: JohnMikkelsonOvative Date: Wed, 24 Apr 2024 17:06:01 -0500 Subject: [PATCH 02/20] EMRGE-1446 Unit Tests for Selectable Auth all passing, 205 passed with 11 warnings --- .../unit_tests/integration/config.py | 7 +++++-- .../unit_tests/integration/request_builder.py | 10 +++++----- .../integration/test_ads_insights_action_product_id.py | 10 +++++----- .../unit_tests/integration/test_videos.py | 4 ++-- .../unit_tests/test_source.py | 5 ++++- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py index c8e65f5085b0a..1f70fdae9d5a2 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py @@ -10,7 +10,7 @@ import pendulum -ACCESS_TOKEN = "test_access_token" +SERVICE_ACCOUNT_INFO = "test_access_token" ACCOUNT_ID = "111111111111111" CLIENT_ID = "test_client_id" CLIENT_SECRET = "test_client_secret" @@ -25,7 +25,10 @@ class ConfigBuilder: def __init__(self) -> None: self._config: MutableMapping[str, Any] = { "account_ids": [ACCOUNT_ID], - "access_token": ACCESS_TOKEN, + "credentials": { + "auth_type": "Service", + "service_account_info": SERVICE_ACCOUNT_INFO, + }, "start_date": START_DATE, "end_date": END_DATE, "include_deleted": True, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py index ed523146085b5..c721a444e96ba 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py @@ -9,23 +9,23 @@ from airbyte_cdk.test.mock_http.request import HttpRequest -from .config import ACCESS_TOKEN, ACCOUNT_ID +from .config import SERVICE_ACCOUNT_INFO, ACCOUNT_ID def get_account_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: - return RequestBuilder.get_account_endpoint(access_token=ACCESS_TOKEN, account_id=account_id) + return RequestBuilder.get_account_endpoint(access_token=SERVICE_ACCOUNT_INFO, account_id=account_id) def get_ads_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: - return RequestBuilder.get_ad_endpoint(access_token=ACCESS_TOKEN, account_id=account_id) + return RequestBuilder.get_ad_endpoint(access_token=SERVICE_ACCOUNT_INFO, account_id=account_id) def get_campaigns_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: - return RequestBuilder.get_campaign_endpoint(access_token=ACCESS_TOKEN, account_id=account_id) + return RequestBuilder.get_campaign_endpoint(access_token=SERVICE_ACCOUNT_INFO, account_id=account_id) def get_ad_sets_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: - return RequestBuilder.get_ad_sets_endpoint(access_token=ACCESS_TOKEN, account_id=account_id) + return RequestBuilder.get_ad_sets_endpoint(access_token=SERVICE_ACCOUNT_INFO, account_id=account_id) class RequestBuilder: diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py index 3fdcc12f0860d..ad3b4410d3aa5 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py @@ -25,7 +25,7 @@ from airbyte_protocol.models import AirbyteStateMessage, StreamDescriptor, SyncMode from source_facebook_marketing.streams.async_job import Status -from .config import ACCESS_TOKEN, ACCOUNT_ID, DATE_FORMAT, END_DATE, NOW, START_DATE, ConfigBuilder +from .config import SERVICE_ACCOUNT_INFO, ACCOUNT_ID, DATE_FORMAT, END_DATE, NOW, START_DATE, ConfigBuilder from .pagination import NEXT_PAGE_TOKEN, FacebookMarketingPaginationStrategy from .request_builder import RequestBuilder, get_account_request from .response_builder import build_response, error_reduce_amount_of_data_response, get_account_response @@ -38,7 +38,7 @@ def _update_api_throttle_limit_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: - return RequestBuilder.get_insights_endpoint(access_token=ACCESS_TOKEN, account_id=account_id) + return RequestBuilder.get_insights_endpoint(access_token=SERVICE_ACCOUNT_INFO, account_id=account_id) def _job_start_request( @@ -161,18 +161,18 @@ def _job_start_request( "action_attribution_windows": ["1d_click", "7d_click", "28d_click", "1d_view", "7d_view", "28d_view"], "time_range": {"since": since, "until": until}, } - return RequestBuilder.get_insights_endpoint(access_token=ACCESS_TOKEN, account_id=account_id).with_body(encode_request_body(body)) + return RequestBuilder.get_insights_endpoint(access_token=SERVICE_ACCOUNT_INFO, account_id=account_id).with_body(encode_request_body(body)) def _job_status_request(report_run_ids: Union[str, List[str]]) -> RequestBuilder: if isinstance(report_run_ids, str): report_run_ids = [report_run_ids] body = {"batch": [{"method": "GET", "relative_url": f"{report_run_id}/"} for report_run_id in report_run_ids]} - return RequestBuilder.get_execute_batch_endpoint(access_token=ACCESS_TOKEN).with_body(encode_request_body(body)) + return RequestBuilder.get_execute_batch_endpoint(access_token=SERVICE_ACCOUNT_INFO).with_body(encode_request_body(body)) def _get_insights_request(job_id: str) -> RequestBuilder: - return RequestBuilder.get_insights_download_endpoint(access_token=ACCESS_TOKEN, job_id=job_id).with_limit(100) + return RequestBuilder.get_insights_download_endpoint(access_token=SERVICE_ACCOUNT_INFO, job_id=job_id).with_limit(100) def _update_api_throttle_limit_response(api_throttle: Optional[int] = 0) -> HttpResponse: diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py index ee814fd201f0b..588b26d0037fd 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py @@ -20,7 +20,7 @@ from airbyte_cdk.test.state_builder import StateBuilder from airbyte_protocol.models import AirbyteStateMessage, SyncMode -from .config import ACCESS_TOKEN, ACCOUNT_ID, NOW, ConfigBuilder +from .config import SERVICE_ACCOUNT_INFO, ACCOUNT_ID, NOW, ConfigBuilder from .pagination import NEXT_PAGE_TOKEN, FacebookMarketingPaginationStrategy from .request_builder import RequestBuilder, get_account_request from .response_builder import error_reduce_amount_of_data_response, get_account_response @@ -63,7 +63,7 @@ def _get_videos_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: return ( - RequestBuilder.get_videos_endpoint(access_token=ACCESS_TOKEN, account_id=account_id) + RequestBuilder.get_videos_endpoint(access_token=SERVICE_ACCOUNT_INFO, account_id=account_id) .with_limit(100) .with_fields(_FIELDS) .with_summary() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py index 202c1ce1fd67e..21b8bf22f8ded 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py @@ -28,7 +28,10 @@ def config_fixture(requests_mock): config = { "account_ids": ["123"], - "access_token": "TOKEN", + "credentials": { + "auth_type": "Service", + "service_account_info": "SERVICE_ACCOUNT_INFO", + }, "start_date": "2019-10-10T00:00:00Z", "end_date": "2020-10-10T00:00:00Z", } From ccd385a2f44419914cb0752f0cab8b9d1faaaf64 Mon Sep 17 00:00:00 2001 From: JohnMikkelsonOvative Date: Fri, 26 Apr 2024 09:49:16 -0500 Subject: [PATCH 03/20] EMRGE-1446 Changed naming to reflect Facebook standard, updated sample config. Updated version to reflect Airbyte versioning standard (this use case is a major version upgrade due to configuration of credentials needed to be changed) --- .../connectors/source-facebook-marketing/metadata.yaml | 2 +- .../connectors/source-facebook-marketing/pyproject.toml | 2 +- .../sample_files/sample_config.json | 5 ++++- .../source_facebook_marketing/spec.py | 9 ++++++--- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml index 8edca72a35a48..99dd0efd17d16 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c - dockerImageTag: 2.1.5 + dockerImageTag: 3.0.0 dockerRepository: airbyte/source-facebook-marketing documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing githubIssueLabel: source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml b/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml index 96789204fc0ec..2b1a774c1559d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml +++ b/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "2.1.5" +version = "3.0.0" name = "source-facebook-marketing" description = "Source implementation for Facebook Marketing." authors = [ "Airbyte ",] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/sample_files/sample_config.json b/airbyte-integrations/connectors/source-facebook-marketing/sample_files/sample_config.json index f69ba65fd2672..4a3682a4f9da0 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/sample_files/sample_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/sample_files/sample_config.json @@ -2,5 +2,8 @@ "start_date": "2020-09-25T00:00:00Z", "end_date": "2021-01-01T00:00:00Z", "account_id": "", - "access_token": "" + "credentials": { + "auth_type": "Service", + "service_account_info": "" + } } diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index 6a53d9c11002d..d78a0c4ad9851 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -41,12 +41,12 @@ class Config(OneOfOptionConfig): ) client_secret: str = Field( title="Client Secret", - description="Client Secret for the Google Drive API", + description="Client Secret for the Facebook Marketing API", airbyte_secret=True, ) refresh_token: str = Field( title="Refresh Token", - description="Refresh Token for the Google Drive API", + description="Refresh Token for the Facebook Marketing API", airbyte_secret=True, ) @@ -59,7 +59,10 @@ class Config(OneOfOptionConfig): auth_type: Literal["Service"] = Field("Service", const=True) service_account_info: str = Field( title="Service Account Information", - description='The JSON key of the service account to use for authorization.', + description="The value of the generated access token. " + 'From your App’s Dashboard, click on "Marketing API" then "Tools". ' + 'Select permissions ads_management, ads_read, read_insights, business_management. Then click on "Get token". ' + 'See the docs for more information.', airbyte_secret=True, ) From dba37487d03f11857888c7b7b608963871dd3bbb Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Thu, 16 May 2024 15:35:13 -0600 Subject: [PATCH 04/20] FB OAuth switch --- .../config_migrations.py | 69 +++++++++++++++++ .../source_facebook_marketing/run.py | 3 +- .../source_facebook_marketing/source.py | 12 +-- .../source_facebook_marketing/spec.py | 16 ++-- .../unit_tests/integration/request_builder.py | 2 +- .../test_ads_insights_action_product_id.py | 2 +- .../unit_tests/integration/test_videos.py | 2 +- .../unit_tests/test_config_migrations.py | 75 ++++++++++++++++++- .../test_new_access_token_config.json | 16 ++++ .../test_new_client_config.json | 16 ++++ .../test_old_access_token_config.json | 14 ++++ .../test_old_client_config.json | 14 ++++ 12 files changed, 220 insertions(+), 21 deletions(-) create mode 100644 airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_access_token_config.json create mode 100644 airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_client_config.json create mode 100644 airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json create mode 100644 airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py index 8ca9289c97e28..512b8b41c05eb 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py @@ -121,3 +121,72 @@ def transform(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: config[stream_filter] = statuses # return transformed config return config + + +class MigrateSecretsPathInConnector: + """ + This class stands for migrating the config at runtime. + This migration is intended for backwards compatibility with the previous version, so existing secrets configurations gets migrated to new path. + + Starting from `X.X.X`, the `client_id`, `client_secret` and `access_token` will be placed at `credentials` path. + """ + + @classmethod + def should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether the config should be migrated to nest existing fields at credentials. + It is assumed if credentials does not exist on configuration, `client_id`, `client_secret` and `access_token` exists on root path. + Returns: + > True, if the migration is necessary + > False, otherwise. + """ + credentials = config.get("credentials", None) + return credentials is None or not all(key in credentials for key in ["client_id", "client_secret", "access_token"]) + + @classmethod + def migrate(cls, args: List[str], source: Source) -> None: + """ + This method checks the input args, should the config be migrated, + transform if neccessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls.should_migrate(config): + cls.emit_control_message( + cls.modify_and_save(config_path, source, config), + ) + + @classmethod + def transform(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: + # transform the config + config["credentials"] = {} + if "client_id" in config: + config["credentials"]["client_id"] = config.pop("client_id") + if "client_secret" in config: + config["credentials"]["client_secret"] = config.pop("client_secret") + if "access_token" in config: + config["credentials"]["access_token"] = config.pop("access_token") + # return transformed config + return config + + @classmethod + def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls.transform(config) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository._message_queue: + print(message.json(exclude_unset=True)) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/run.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/run.py index b070561488f09..ecb5dcd5cbad6 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/run.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/run.py @@ -7,7 +7,7 @@ from airbyte_cdk.entrypoint import launch -from .config_migrations import MigrateAccountIdToArray, MigrateIncludeDeletedToStatusFilters +from .config_migrations import MigrateAccountIdToArray, MigrateIncludeDeletedToStatusFilters, MigrateSecretsPathInConnector from .source import SourceFacebookMarketing @@ -15,4 +15,5 @@ def run(): source = SourceFacebookMarketing() MigrateAccountIdToArray.migrate(sys.argv[1:], source) MigrateIncludeDeletedToStatusFilters.migrate(sys.argv[1:], source) + MigrateSecretsPathInConnector.migrate(sys.argv[1:], source) launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 1dd0051b1efe3..a40e23b6854eb 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -95,9 +95,9 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> if config.start_date and config.end_date < config.start_date: return False, "End date must be equal or after start date." - if config.credentials.auth_type == 'Client': - api = API(access_token=config.credentials.refresh_token, page_size=config.page_size) - if config.credentials.auth_type == 'Service': + if config.credentials.auth_type == "Client": + api = API(access_token=config.credentials.access_token, page_size=config.page_size) + if config.credentials.auth_type == "Service": api = API(access_token=config.credentials.service_account_info, page_size=config.page_size) for account_id in config.account_ids: @@ -132,9 +132,9 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: config.start_date = validate_start_date(config.start_date) config.end_date = validate_end_date(config.start_date, config.end_date) - if config.credentials.auth_type == 'Client': - api = API(access_token=config.credentials.refresh_token, page_size=config.page_size) - if config.credentials.auth_type == 'Service': + if config.credentials.auth_type == "Client": + api = API(access_token=config.credentials.access_token, page_size=config.page_size) + if config.credentials.auth_type == "Service": api = API(access_token=config.credentials.service_account_info, page_size=config.page_size) # if start_date not specified then set default start_date for report streams to 2 years ago diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index d78a0c4ad9851..a5cb139b74899 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -5,7 +5,7 @@ import logging from datetime import datetime, timezone from enum import Enum -from typing import List, Optional, Set, Literal, Union +from typing import List, Literal, Optional, Set, Union from airbyte_cdk.sources.config import BaseConfig from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig @@ -44,11 +44,6 @@ class Config(OneOfOptionConfig): description="Client Secret for the Facebook Marketing API", airbyte_secret=True, ) - refresh_token: str = Field( - title="Refresh Token", - description="Refresh Token for the Facebook Marketing API", - airbyte_secret=True, - ) class ServiceAccountCredentials(BaseModel): @@ -60,12 +55,13 @@ class Config(OneOfOptionConfig): service_account_info: str = Field( title="Service Account Information", description="The value of the generated access token. " - 'From your App’s Dashboard, click on "Marketing API" then "Tools". ' - 'Select permissions ads_management, ads_read, read_insights, business_management. Then click on "Get token". ' - 'See the docs for more information.', + 'From your App’s Dashboard, click on "Marketing API" then "Tools". ' + 'Select permissions ads_management, ads_read, read_insights, business_management. Then click on "Get token". ' + 'See the docs for more information.', airbyte_secret=True, ) + class InsightConfig(BaseModel): """Config for custom insights""" @@ -181,7 +177,7 @@ class Config: title="Authentication", description="Credentials for connecting to the Facebook Marketing API", discriminator="auth_type", - type="object" + type="object", ) start_date: Optional[datetime] = Field( diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py index c721a444e96ba..a0791f5a9d7e6 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py @@ -9,7 +9,7 @@ from airbyte_cdk.test.mock_http.request import HttpRequest -from .config import SERVICE_ACCOUNT_INFO, ACCOUNT_ID +from .config import ACCOUNT_ID, SERVICE_ACCOUNT_INFO def get_account_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py index ad3b4410d3aa5..736668bd812f1 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py @@ -25,7 +25,7 @@ from airbyte_protocol.models import AirbyteStateMessage, StreamDescriptor, SyncMode from source_facebook_marketing.streams.async_job import Status -from .config import SERVICE_ACCOUNT_INFO, ACCOUNT_ID, DATE_FORMAT, END_DATE, NOW, START_DATE, ConfigBuilder +from .config import ACCOUNT_ID, DATE_FORMAT, END_DATE, NOW, SERVICE_ACCOUNT_INFO, START_DATE, ConfigBuilder from .pagination import NEXT_PAGE_TOKEN, FacebookMarketingPaginationStrategy from .request_builder import RequestBuilder, get_account_request from .response_builder import build_response, error_reduce_amount_of_data_response, get_account_response diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py index 588b26d0037fd..b3f4d90fc81ef 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py @@ -20,7 +20,7 @@ from airbyte_cdk.test.state_builder import StateBuilder from airbyte_protocol.models import AirbyteStateMessage, SyncMode -from .config import SERVICE_ACCOUNT_INFO, ACCOUNT_ID, NOW, ConfigBuilder +from .config import ACCOUNT_ID, NOW, SERVICE_ACCOUNT_INFO, ConfigBuilder from .pagination import NEXT_PAGE_TOKEN, FacebookMarketingPaginationStrategy from .request_builder import RequestBuilder, get_account_request from .response_builder import error_reduce_amount_of_data_response, get_account_response diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py index 759eb6e8f8485..dbc138796f44e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py @@ -10,7 +10,11 @@ import pytest from airbyte_cdk.models import OrchestratorType, Type from airbyte_cdk.sources import Source -from source_facebook_marketing.config_migrations import MigrateAccountIdToArray, MigrateIncludeDeletedToStatusFilters +from source_facebook_marketing.config_migrations import ( + MigrateAccountIdToArray, + MigrateIncludeDeletedToStatusFilters, + MigrateSecretsPathInConnector, +) from source_facebook_marketing.source import SourceFacebookMarketing # BASE ARGS @@ -19,6 +23,7 @@ _EXCLUDE_DELETE_CONFIGS_PATH = "test_migrations/include_deleted_to_status_filters/include_deleted_false" _INCLUDE_DELETE_CONFIGS_PATH = "test_migrations/include_deleted_to_status_filters/include_deleted_true" _ACCOUNT_ID_TO_ARRAY_CONFIGS_PATH = "test_migrations/account_id_to_array" +_SECRETS_TO_CREDENTIALS_CONFIGS_PATH = "test_migrations/secrets_to_credentials" def load_config(config_path: str) -> Mapping[str, Any]: @@ -162,3 +167,71 @@ def test_should_not_migrate_upgraded_config(self): new_config = load_config(self.UPGRADED_TEST_CONFIG_PATH) migration_instance = MigrateIncludeDeletedToStatusFilters() assert not migration_instance.should_migrate(new_config) + +class TestMigrateSecretsPathInConnector: + OLD_TEST_CONFIG_PATH_ACCESS_TOKEN = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_old_access_token_config.json") + NEW_TEST_CONFIG_PATH_ACCESS_TOKEN = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_new_access_token_config.json") + OLD_TEST_CONFIG_PATH_ACCESS_TOKEN = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_old_client_config.json") + NEW_TEST_CONFIG_PATH_ACCESS_TOKEN = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_new_client_config.json") + + @staticmethod + def revert_migration(config_path: str = OLD_TEST_CONFIG_PATH_ACCESS_TOKEN) -> None: + with open(config_path, "r") as test_config: + config = json.load(test_config) + credentials = config.pop("credentials") + with open(config_path, "w") as updated_config: + config = json.dumps({**config, **credentials}) + updated_config.write(config) + + def test_migrate_access_token_config(self): + migration_instance = MigrateSecretsPathInConnector() + original_config = load_config(self.OLD_TEST_CONFIG_PATH_ACCESS_TOKEN) + # migrate the test_config + migration_instance.migrate([CMD, "--config", self.OLD_TEST_CONFIG_PATH_ACCESS_TOKEN], SOURCE) + # load the updated config + test_migrated_config = load_config(self.OLD_TEST_CONFIG_PATH_ACCESS_TOKEN) + # check migrated property + assert "credentials" in test_migrated_config + assert isinstance(test_migrated_config["credentials"], dict) + credentials = test_migrated_config["credentials"] + assert "access_token" in credentials + # check the migration should be skipped, once already done + assert not migration_instance.should_migrate(test_migrated_config) + # load the old custom reports VS migrated + assert original_config["access_token"] == credentials["access_token"] + # test CONTROL MESSAGE was emitted + control_msg = migration_instance.message_repository._message_queue[0] + assert control_msg.type == Type.CONTROL + assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG + # revert the test_config to the starting point + self.revert_migration() + + def test_migrate_client_config(self): + migration_instance = MigrateSecretsPathInConnector() + original_config = load_config(self.OLD_TEST_CONFIG_PATH_CLIENT) + # migrate the test_config + migration_instance.migrate([CMD, "--config", self.OLD_TEST_CONFIG_PATH_CLIENT], SOURCE) + # load the updated config + test_migrated_config = load_config(self.OLD_TEST_CONFIG_PATH_CLIENT) + # check migrated property + assert "credentials" in test_migrated_config + assert isinstance(test_migrated_config["credentials"], dict) + credentials = test_migrated_config["credentials"] + assert "client_id" in credentials + assert "client_secret" in credentials + # check the migration should be skipped, once already done + assert not migration_instance.should_migrate(test_migrated_config) + # load the old custom reports VS migrated + assert original_config["client_id"] == credentials["client_id"] + assert original_config["client_secret"] == credentials["client_secret"] + # test CONTROL MESSAGE was emitted + control_msg = migration_instance.message_repository._message_queue[0] + assert control_msg.type == Type.CONTROL + assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG + # revert the test_config to the starting point + self.revert_migration() + + def test_should_not_migrate_new_config(self): + new_config = load_config(self.NEW_TEST_CONFIG_PATH_CLIENT) + migration_instance = MigrateSecretsPathInConnector() + assert not migration_instance.should_migrate(new_config) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_access_token_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_access_token_config.json new file mode 100644 index 0000000000000..6ebf309b081ba --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_access_token_config.json @@ -0,0 +1,16 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_ids": ["01234567890"], + "credentials": { + "access_token": "access_token" + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_client_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_client_config.json new file mode 100644 index 0000000000000..6ebf309b081ba --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_client_config.json @@ -0,0 +1,16 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_ids": ["01234567890"], + "credentials": { + "access_token": "access_token" + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json new file mode 100644 index 0000000000000..a04560eb77103 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json @@ -0,0 +1,14 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_id": "01234567890", + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json new file mode 100644 index 0000000000000..a04560eb77103 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json @@ -0,0 +1,14 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_id": "01234567890", + "access_token": "access_token" +} From 040e234b0e2131756cd7f3b68e0ce218e662ca24 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Sat, 18 May 2024 14:44:36 -0600 Subject: [PATCH 05/20] fixing unit test --- .../config_migrations.py | 6 ++++-- .../unit_tests/test_config_migrations.py | 19 ++++++++++++------- .../account_id_to_array/test_old_config.json | 15 +-------------- .../include_deleted_true/test_old_config.json | 16 +--------------- .../test_new_client_config.json | 3 ++- .../test_old_access_token_config.json | 15 +-------------- .../test_old_client_config.json | 15 +-------------- 7 files changed, 22 insertions(+), 67 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py index 512b8b41c05eb..69faca413709c 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py @@ -128,9 +128,11 @@ class MigrateSecretsPathInConnector: This class stands for migrating the config at runtime. This migration is intended for backwards compatibility with the previous version, so existing secrets configurations gets migrated to new path. - Starting from `X.X.X`, the `client_id`, `client_secret` and `access_token` will be placed at `credentials` path. + Starting from `3.0.0`, the `client_id`, `client_secret` and `access_token` will be placed at `credentials` path. """ + message_repository: MessageRepository = InMemoryMessageRepository() + @classmethod def should_migrate(cls, config: Mapping[str, Any]) -> bool: """ @@ -141,7 +143,7 @@ def should_migrate(cls, config: Mapping[str, Any]) -> bool: > False, otherwise. """ credentials = config.get("credentials", None) - return credentials is None or not all(key in credentials for key in ["client_id", "client_secret", "access_token"]) + return credentials is None or ("access_token" not in credentials and not ("client_id" in credentials and "client_id" in credentials)) @classmethod def migrate(cls, args: List[str], source: Source) -> None: diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py index dbc138796f44e..8a99bc97995c7 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py @@ -171,14 +171,14 @@ def test_should_not_migrate_upgraded_config(self): class TestMigrateSecretsPathInConnector: OLD_TEST_CONFIG_PATH_ACCESS_TOKEN = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_old_access_token_config.json") NEW_TEST_CONFIG_PATH_ACCESS_TOKEN = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_new_access_token_config.json") - OLD_TEST_CONFIG_PATH_ACCESS_TOKEN = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_old_client_config.json") - NEW_TEST_CONFIG_PATH_ACCESS_TOKEN = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_new_client_config.json") + OLD_TEST_CONFIG_PATH_CLIENT = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_old_client_config.json") + NEW_TEST_CONFIG_PATH_CLIENT = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_new_client_config.json") @staticmethod - def revert_migration(config_path: str = OLD_TEST_CONFIG_PATH_ACCESS_TOKEN) -> None: + def revert_migration(config_path: str) -> None: with open(config_path, "r") as test_config: config = json.load(test_config) - credentials = config.pop("credentials") + credentials = config.pop("credentials",{}) with open(config_path, "w") as updated_config: config = json.dumps({**config, **credentials}) updated_config.write(config) @@ -204,7 +204,7 @@ def test_migrate_access_token_config(self): assert control_msg.type == Type.CONTROL assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG # revert the test_config to the starting point - self.revert_migration() + self.revert_migration(self.OLD_TEST_CONFIG_PATH_ACCESS_TOKEN) def test_migrate_client_config(self): migration_instance = MigrateSecretsPathInConnector() @@ -229,9 +229,14 @@ def test_migrate_client_config(self): assert control_msg.type == Type.CONTROL assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG # revert the test_config to the starting point - self.revert_migration() + self.revert_migration(self.OLD_TEST_CONFIG_PATH_CLIENT) - def test_should_not_migrate_new_config(self): + def test_should_not_migrate_new_client_config(self): new_config = load_config(self.NEW_TEST_CONFIG_PATH_CLIENT) migration_instance = MigrateSecretsPathInConnector() assert not migration_instance.should_migrate(new_config) + + def test_should_not_migrate_new_access_token_config(self): + new_config = load_config(self.NEW_TEST_CONFIG_PATH_ACCESS_TOKEN) + migration_instance = MigrateSecretsPathInConnector() + assert not migration_instance.should_migrate(new_config) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json index a04560eb77103..526c0c0caa7ef 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json @@ -1,14 +1 @@ -{ - "start_date": "2021-02-08T00:00:00Z", - "end_date": "2021-02-15T00:00:00Z", - "custom_insights": [ - { - "name": "custom_insight_stream", - "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], - "breakdowns": ["gender"], - "action_breakdowns": [] - } - ], - "account_id": "01234567890", - "access_token": "access_token" -} +{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "account_id": "01234567890", "access_token": "access_token"} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json index 0cf00a31758d4..52234b6754295 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json @@ -1,15 +1 @@ -{ - "start_date": "2021-02-08T00:00:00Z", - "end_date": "2021-02-15T00:00:00Z", - "custom_insights": [ - { - "name": "custom_insight_stream", - "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], - "breakdowns": ["gender"], - "action_breakdowns": [] - } - ], - "include_deleted": true, - "account_ids": ["01234567890"], - "access_token": "access_token" -} +{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "include_deleted": true, "account_ids": ["01234567890"], "access_token": "access_token"} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_client_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_client_config.json index 6ebf309b081ba..a8ebfdf994b93 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_client_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_client_config.json @@ -11,6 +11,7 @@ ], "account_ids": ["01234567890"], "credentials": { - "access_token": "access_token" + "client_id": "client_id", + "client_secret": "client_secret" } } diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json index a04560eb77103..526c0c0caa7ef 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json @@ -1,14 +1 @@ -{ - "start_date": "2021-02-08T00:00:00Z", - "end_date": "2021-02-15T00:00:00Z", - "custom_insights": [ - { - "name": "custom_insight_stream", - "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], - "breakdowns": ["gender"], - "action_breakdowns": [] - } - ], - "account_id": "01234567890", - "access_token": "access_token" -} +{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "account_id": "01234567890", "access_token": "access_token"} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json index a04560eb77103..0bec626ea9da4 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json @@ -1,14 +1 @@ -{ - "start_date": "2021-02-08T00:00:00Z", - "end_date": "2021-02-15T00:00:00Z", - "custom_insights": [ - { - "name": "custom_insight_stream", - "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], - "breakdowns": ["gender"], - "action_breakdowns": [] - } - ], - "account_id": "01234567890", - "access_token": "access_token" -} +{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "account_id": "01234567890", "client_id": "client_id", "client_secret": "client_secret"} \ No newline at end of file From 7aa761e01068d0cacd44ec3e6517d37028ef13d2 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Sat, 18 May 2024 14:55:57 -0600 Subject: [PATCH 06/20] fixing formatting --- .../config_migrations.py | 4 +++- .../account_id_to_array/test_old_config.json | 15 ++++++++++++++- .../include_deleted_true/test_old_config.json | 16 +++++++++++++++- .../test_old_access_token_config.json | 15 ++++++++++++++- .../test_old_client_config.json | 16 +++++++++++++++- 5 files changed, 61 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py index 69faca413709c..6ac766bc23495 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py @@ -143,7 +143,9 @@ def should_migrate(cls, config: Mapping[str, Any]) -> bool: > False, otherwise. """ credentials = config.get("credentials", None) - return credentials is None or ("access_token" not in credentials and not ("client_id" in credentials and "client_id" in credentials)) + return credentials is None or ( + "access_token" not in credentials and not ("client_id" in credentials and "client_id" in credentials) + ) @classmethod def migrate(cls, args: List[str], source: Source) -> None: diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json index 526c0c0caa7ef..a04560eb77103 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json @@ -1 +1,14 @@ -{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "account_id": "01234567890", "access_token": "access_token"} \ No newline at end of file +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_id": "01234567890", + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json index 52234b6754295..0cf00a31758d4 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json @@ -1 +1,15 @@ -{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "include_deleted": true, "account_ids": ["01234567890"], "access_token": "access_token"} \ No newline at end of file +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "include_deleted": true, + "account_ids": ["01234567890"], + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json index 526c0c0caa7ef..a04560eb77103 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json @@ -1 +1,14 @@ -{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "account_id": "01234567890", "access_token": "access_token"} \ No newline at end of file +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_id": "01234567890", + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json index 0bec626ea9da4..eee9ac482f9eb 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json @@ -1 +1,15 @@ -{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "account_id": "01234567890", "client_id": "client_id", "client_secret": "client_secret"} \ No newline at end of file +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_id": "01234567890", + "client_id": "client_id", + "client_secret": "client_secret" +} From e5f8341f996b21395b30d4af974c480cc6df6a7f Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 21 May 2024 12:43:48 -0600 Subject: [PATCH 07/20] modifying specs to test pass --- .../tests/fixtures/cloud_registry.json | 37 ++++++++++--- .../tests/fixtures/oss_registry.json | 37 ++++++++++--- .../integration_tests/spec.json | 53 +++++++++++-------- .../source-facebook-marketing/metadata.yaml | 3 ++ .../sources/facebook-marketing-migrations.md | 4 ++ .../sources/facebook-marketing.md | 1 + 6 files changed, 102 insertions(+), 33 deletions(-) diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/cloud_registry.json b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/cloud_registry.json index dc87958945887..c95e9544e86ef 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/cloud_registry.json +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/cloud_registry.json @@ -8778,12 +8778,37 @@ "examples": ["2017-01-26T00:00:00Z"], "type": "string" }, - "access_token": { - "title": "Access Token", - "description": "The value of the generated access token. From your App’s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", - "order": 3, - "airbyte_secret": true, - "type": "string" + "credentials": { + "title": "authentication credentials", + "description": "", + "type": "object", + "properties": { + "access_token": { + "title": "Access Token", + "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", + "order": 3, + "airbyte_secret": true, + "type": "string" + }, + "client_id": { + "title": "Client Id", + "description": "The Client Id for your OAuth app", + "airbyte_secret": true, + "type": "string" + }, + "client_secret": { + "title": "Client Secret", + "description": "The Client Secret for your OAuth app", + "airbyte_secret": true, + "type": "string" + }, + "auth_type": { + "title": "Auth Type", + "description": "The type of authentication to use. If you are using an OAuth app, select 'oauth'. If you are using an access token, select 'access_token'.", + "enum": ["Service", "Client"], + "type": "string" + } + } }, "include_deleted": { "title": "Include Deleted Campaigns, Ads, and AdSets", diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json index a1271847cddbe..0460940b6d45d 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json @@ -13214,12 +13214,37 @@ "examples": ["2017-01-26T00:00:00Z"], "type": "string" }, - "access_token": { - "title": "Access Token", - "description": "The value of the generated access token. From your App’s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", - "order": 3, - "airbyte_secret": true, - "type": "string" + "credentials": { + "title": "authentication credentials", + "description": "", + "type": "object", + "properties": { + "access_token": { + "title": "Access Token", + "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", + "order": 3, + "airbyte_secret": true, + "type": "string" + }, + "client_id": { + "title": "Client Id", + "description": "The Client Id for your OAuth app", + "airbyte_secret": true, + "type": "string" + }, + "client_secret": { + "title": "Client Secret", + "description": "The Client Secret for your OAuth app", + "airbyte_secret": true, + "type": "string" + }, + "auth_type": { + "title": "Auth Type", + "description": "The type of authentication to use. If you are using an OAuth app, select 'oauth'. If you are using an access token, select 'access_token'.", + "enum": ["Service", "Client"], + "type": "string" + } + } }, "include_deleted": { "title": "Include Deleted Campaigns, Ads, and AdSets", diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index 328659d53acce..de74a2ea77405 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -19,12 +19,37 @@ }, "uniqueItems": true }, - "access_token": { - "title": "Access Token", - "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", - "order": 1, - "airbyte_secret": true, - "type": "string" + "credentials": { + "title": "authentication credentials", + "description": "", + "type": "object", + "properties": { + "access_token": { + "title": "Access Token", + "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", + "order": 1, + "airbyte_secret": true, + "type": "string" + }, + "client_id": { + "title": "Client Id", + "description": "The Client Id for your OAuth app", + "airbyte_secret": true, + "type": "string" + }, + "client_secret": { + "title": "Client Secret", + "description": "The Client Secret for your OAuth app", + "airbyte_secret": true, + "type": "string" + }, + "auth_type": { + "title": "Auth Type", + "description": "The type of authentication to use. If you are using an OAuth app, select 'oauth'. If you are using an access token, select 'access_token'.", + "enum": ["Service", "Client"], + "type": "string" + } + } }, "start_date": { "title": "Start Date", @@ -448,23 +473,9 @@ "default": true, "airbyte_hidden": true, "type": "boolean" - }, - "client_id": { - "title": "Client Id", - "description": "The Client Id for your OAuth app", - "airbyte_secret": true, - "airbyte_hidden": true, - "type": "string" - }, - "client_secret": { - "title": "Client Secret", - "description": "The Client Secret for your OAuth app", - "airbyte_secret": true, - "airbyte_hidden": true, - "type": "string" } }, - "required": ["account_ids", "access_token"] + "required": ["account_ids", "credentials"] }, "supportsIncremental": true, "supported_destination_sync_modes": ["append"], diff --git a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml index 01121a422cabe..439dff01ee3c8 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml @@ -44,6 +44,9 @@ data: - "ads_insights_region" - "ads_insights_dma" - "ads_insights_action_product_id" + 3.0.0: + message: "Credentials schema has been updated. Migration will be required to ensure backward compatibility. For more information [visit](https://docs.airbyte.com/integrations/sources/facebook-marketing-migrations)" + upgradeDeadline: "2024-05-18" suggestedStreams: streams: - ads_insights diff --git a/docs/integrations/sources/facebook-marketing-migrations.md b/docs/integrations/sources/facebook-marketing-migrations.md index 77fe1a1f51727..42231aede987f 100644 --- a/docs/integrations/sources/facebook-marketing-migrations.md +++ b/docs/integrations/sources/facebook-marketing-migrations.md @@ -1,5 +1,9 @@ # Facebook Marketing Migration Guide +## Upgrading to 3.0.0 + +Support both old and new way of authentication to cover the existing Customers by applying the config migration, see the example in the /config_migrations.py + ## Upgrading to 2.0.0 Streams Ads-Insights-\* streams now have updated schemas. diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index ade3523b7da7f..385609faf8067 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -202,6 +202,7 @@ The Facebook Marketing connector uses the `lookback_window` parameter to repeate | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.0.0 | 2024-05-18 | [37625](https://github.com/airbytehq/airbyte/pull/37625) | Source Facebook-Marketing: Add Selectable Auth | | 2.1.9 | 2024-05-17 | [38301](https://github.com/airbytehq/airbyte/pull/38301) | Fix data inaccuracies when `wish_bid` is requested | | 2.1.8 | 2024-05-07 | [37771](https://github.com/airbytehq/airbyte/pull/37771) | Handle errors without API error codes/messages | | 2.1.7 | 2024-04-24 | [36634](https://github.com/airbytehq/airbyte/pull/36634) | Update to CDK 0.80.0 | From 8331b495360722daed4dbc0fefb441dd88eae34a Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Thu, 23 May 2024 06:40:00 -0600 Subject: [PATCH 08/20] applying feedback --- .../source-facebook-marketing/metadata.yaml | 5 +---- .../source-facebook-marketing/pyproject.toml | 2 +- .../sample_files/sample_config.json | 2 +- .../source_facebook_marketing/config_migrations.py | 14 +++++++++----- .../source_facebook_marketing/source.py | 7 ++----- .../source_facebook_marketing/spec.py | 4 ++-- .../unit_tests/integration/config.py | 4 ++-- .../unit_tests/integration/request_builder.py | 10 +++++----- .../test_ads_insights_action_product_id.py | 10 +++++----- .../unit_tests/integration/test_videos.py | 4 ++-- .../unit_tests/test_source.py | 2 +- .../sources/facebook-marketing-migrations.md | 4 ---- docs/integrations/sources/facebook-marketing.md | 2 +- 13 files changed, 32 insertions(+), 38 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml index 439dff01ee3c8..779eb3c61fe6b 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c - dockerImageTag: 3.0.0 + dockerImageTag: 2.2.0 dockerRepository: airbyte/source-facebook-marketing documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing githubIssueLabel: source-facebook-marketing @@ -44,9 +44,6 @@ data: - "ads_insights_region" - "ads_insights_dma" - "ads_insights_action_product_id" - 3.0.0: - message: "Credentials schema has been updated. Migration will be required to ensure backward compatibility. For more information [visit](https://docs.airbyte.com/integrations/sources/facebook-marketing-migrations)" - upgradeDeadline: "2024-05-18" suggestedStreams: streams: - ads_insights diff --git a/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml b/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml index f40d0f6998976..0d9a6b6336bd3 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml +++ b/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "3.0.0" +version = "2.2.0" name = "source-facebook-marketing" description = "Source implementation for Facebook Marketing." authors = [ "Airbyte ",] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/sample_files/sample_config.json b/airbyte-integrations/connectors/source-facebook-marketing/sample_files/sample_config.json index 4a3682a4f9da0..2ca7d6392db08 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/sample_files/sample_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/sample_files/sample_config.json @@ -4,6 +4,6 @@ "account_id": "", "credentials": { "auth_type": "Service", - "service_account_info": "" + "access_token": "" } } diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py index 6ac766bc23495..be0d7d71e7fd4 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py @@ -128,7 +128,7 @@ class MigrateSecretsPathInConnector: This class stands for migrating the config at runtime. This migration is intended for backwards compatibility with the previous version, so existing secrets configurations gets migrated to new path. - Starting from `3.0.0`, the `client_id`, `client_secret` and `access_token` will be placed at `credentials` path. + Starting from `2.2.0`, the `client_id`, `client_secret` and `access_token` will be placed at `credentials` path. """ message_repository: MessageRepository = InMemoryMessageRepository() @@ -144,7 +144,7 @@ def should_migrate(cls, config: Mapping[str, Any]) -> bool: """ credentials = config.get("credentials", None) return credentials is None or ( - "access_token" not in credentials and not ("client_id" in credentials and "client_id" in credentials) + "access_token" not in credentials and ("client_id" not in credentials or "client_secret" not in credentials) ) @classmethod @@ -168,13 +168,17 @@ def migrate(cls, args: List[str], source: Source) -> None: @classmethod def transform(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: # transform the config - config["credentials"] = {} + if "credentials" not in config: + config["credentials"] = {} + if "access_token" in config: + config["credentials"]["auth_type"] = "Service" + config["credentials"]["access_token"] = config.pop("access_token") if "client_id" in config: + config["credentials"]["auth_type"] = "Client" config["credentials"]["client_id"] = config.pop("client_id") if "client_secret" in config: + config["credentials"]["auth_type"] = "Client" config["credentials"]["client_secret"] = config.pop("client_secret") - if "access_token" in config: - config["credentials"]["access_token"] = config.pop("access_token") # return transformed config return config diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index a40e23b6854eb..95332c4ca1726 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -95,10 +95,7 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> if config.start_date and config.end_date < config.start_date: return False, "End date must be equal or after start date." - if config.credentials.auth_type == "Client": - api = API(access_token=config.credentials.access_token, page_size=config.page_size) - if config.credentials.auth_type == "Service": - api = API(access_token=config.credentials.service_account_info, page_size=config.page_size) + api = API(access_token=config.credentials.access_token, page_size=config.page_size) for account_id in config.account_ids: # Get Ad Account to check creds @@ -135,7 +132,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: if config.credentials.auth_type == "Client": api = API(access_token=config.credentials.access_token, page_size=config.page_size) if config.credentials.auth_type == "Service": - api = API(access_token=config.credentials.service_account_info, page_size=config.page_size) + api = API(access_token=config.credentials.access_token, page_size=config.page_size) # if start_date not specified then set default start_date for report streams to 2 years ago report_start_date = config.start_date or pendulum.now().add(years=-2) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index a5cb139b74899..ad4d800a1c2ab 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -52,8 +52,8 @@ class Config(OneOfOptionConfig): discriminator = "auth_type" auth_type: Literal["Service"] = Field("Service", const=True) - service_account_info: str = Field( - title="Service Account Information", + access_token: str = Field( + title="Access Token", description="The value of the generated access token. " 'From your App’s Dashboard, click on "Marketing API" then "Tools". ' 'Select permissions ads_management, ads_read, read_insights, business_management. Then click on "Get token". ' diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py index 1f70fdae9d5a2..5edc08332f1b1 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py @@ -10,7 +10,7 @@ import pendulum -SERVICE_ACCOUNT_INFO = "test_access_token" +ACCESS_TOKEN = "test_access_token" ACCOUNT_ID = "111111111111111" CLIENT_ID = "test_client_id" CLIENT_SECRET = "test_client_secret" @@ -27,7 +27,7 @@ def __init__(self) -> None: "account_ids": [ACCOUNT_ID], "credentials": { "auth_type": "Service", - "service_account_info": SERVICE_ACCOUNT_INFO, + "ACCESS_TOKEN": ACCESS_TOKEN, }, "start_date": START_DATE, "end_date": END_DATE, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py index a0791f5a9d7e6..c6ed5efddbe98 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py @@ -9,23 +9,23 @@ from airbyte_cdk.test.mock_http.request import HttpRequest -from .config import ACCOUNT_ID, SERVICE_ACCOUNT_INFO +from .config import ACCOUNT_ID, ACCESS_TOKEN def get_account_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: - return RequestBuilder.get_account_endpoint(access_token=SERVICE_ACCOUNT_INFO, account_id=account_id) + return RequestBuilder.get_account_endpoint(access_token=ACCESS_TOKEN, account_id=account_id) def get_ads_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: - return RequestBuilder.get_ad_endpoint(access_token=SERVICE_ACCOUNT_INFO, account_id=account_id) + return RequestBuilder.get_ad_endpoint(access_token=ACCESS_TOKEN, account_id=account_id) def get_campaigns_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: - return RequestBuilder.get_campaign_endpoint(access_token=SERVICE_ACCOUNT_INFO, account_id=account_id) + return RequestBuilder.get_campaign_endpoint(access_token=ACCESS_TOKEN, account_id=account_id) def get_ad_sets_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: - return RequestBuilder.get_ad_sets_endpoint(access_token=SERVICE_ACCOUNT_INFO, account_id=account_id) + return RequestBuilder.get_ad_sets_endpoint(access_token=ACCESS_TOKEN, account_id=account_id) class RequestBuilder: diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py index 21abf6f81f33f..151635f72cb04 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py @@ -25,7 +25,7 @@ from airbyte_protocol.models import AirbyteStateMessage, StreamDescriptor, SyncMode from source_facebook_marketing.streams.async_job import Status -from .config import ACCOUNT_ID, DATE_FORMAT, END_DATE, NOW, SERVICE_ACCOUNT_INFO, START_DATE, ConfigBuilder +from .config import ACCOUNT_ID, DATE_FORMAT, END_DATE, NOW, ACCESS_TOKEN, START_DATE, ConfigBuilder from .pagination import NEXT_PAGE_TOKEN, FacebookMarketingPaginationStrategy from .request_builder import RequestBuilder, get_account_request from .response_builder import build_response, error_reduce_amount_of_data_response, get_account_response @@ -38,7 +38,7 @@ def _update_api_throttle_limit_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: - return RequestBuilder.get_insights_endpoint(access_token=SERVICE_ACCOUNT_INFO, account_id=account_id) + return RequestBuilder.get_insights_endpoint(access_token=ACCESS_TOKEN, account_id=account_id) def _job_start_request( @@ -160,18 +160,18 @@ def _job_start_request( "action_attribution_windows": ["1d_click", "7d_click", "28d_click", "1d_view", "7d_view", "28d_view"], "time_range": {"since": since, "until": until}, } - return RequestBuilder.get_insights_endpoint(access_token=SERVICE_ACCOUNT_INFO, account_id=account_id).with_body(encode_request_body(body)) + return RequestBuilder.get_insights_endpoint(access_token=ACCESS_TOKEN, account_id=account_id).with_body(encode_request_body(body)) def _job_status_request(report_run_ids: Union[str, List[str]]) -> RequestBuilder: if isinstance(report_run_ids, str): report_run_ids = [report_run_ids] body = {"batch": [{"method": "GET", "relative_url": f"{report_run_id}/"} for report_run_id in report_run_ids]} - return RequestBuilder.get_execute_batch_endpoint(access_token=SERVICE_ACCOUNT_INFO).with_body(encode_request_body(body)) + return RequestBuilder.get_execute_batch_endpoint(access_token=ACCESS_TOKEN).with_body(encode_request_body(body)) def _get_insights_request(job_id: str) -> RequestBuilder: - return RequestBuilder.get_insights_download_endpoint(access_token=SERVICE_ACCOUNT_INFO, job_id=job_id).with_limit(100) + return RequestBuilder.get_insights_download_endpoint(access_token=ACCESS_TOKEN, job_id=job_id).with_limit(100) def _update_api_throttle_limit_response(api_throttle: Optional[int] = 0) -> HttpResponse: diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py index b3f4d90fc81ef..d2e425bee2f3a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py @@ -20,7 +20,7 @@ from airbyte_cdk.test.state_builder import StateBuilder from airbyte_protocol.models import AirbyteStateMessage, SyncMode -from .config import ACCOUNT_ID, NOW, SERVICE_ACCOUNT_INFO, ConfigBuilder +from .config import ACCOUNT_ID, NOW, ACCESS_TOKEN, ConfigBuilder from .pagination import NEXT_PAGE_TOKEN, FacebookMarketingPaginationStrategy from .request_builder import RequestBuilder, get_account_request from .response_builder import error_reduce_amount_of_data_response, get_account_response @@ -63,7 +63,7 @@ def _get_videos_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: return ( - RequestBuilder.get_videos_endpoint(access_token=SERVICE_ACCOUNT_INFO, account_id=account_id) + RequestBuilder.get_videos_endpoint(access_token=ACCESS_TOKEN, account_id=account_id) .with_limit(100) .with_fields(_FIELDS) .with_summary() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py index 21b8bf22f8ded..e264760ea701b 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py @@ -30,7 +30,7 @@ def config_fixture(requests_mock): "account_ids": ["123"], "credentials": { "auth_type": "Service", - "service_account_info": "SERVICE_ACCOUNT_INFO", + "access_token": "ACCESS_TOKEN", }, "start_date": "2019-10-10T00:00:00Z", "end_date": "2020-10-10T00:00:00Z", diff --git a/docs/integrations/sources/facebook-marketing-migrations.md b/docs/integrations/sources/facebook-marketing-migrations.md index 42231aede987f..77fe1a1f51727 100644 --- a/docs/integrations/sources/facebook-marketing-migrations.md +++ b/docs/integrations/sources/facebook-marketing-migrations.md @@ -1,9 +1,5 @@ # Facebook Marketing Migration Guide -## Upgrading to 3.0.0 - -Support both old and new way of authentication to cover the existing Customers by applying the config migration, see the example in the /config_migrations.py - ## Upgrading to 2.0.0 Streams Ads-Insights-\* streams now have updated schemas. diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 385609faf8067..e0277a53e5529 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -202,7 +202,7 @@ The Facebook Marketing connector uses the `lookback_window` parameter to repeate | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 3.0.0 | 2024-05-18 | [37625](https://github.com/airbytehq/airbyte/pull/37625) | Source Facebook-Marketing: Add Selectable Auth | +| 2.2.0 | 2024-05-18 | [37625](https://github.com/airbytehq/airbyte/pull/37625) | Source Facebook-Marketing: Add Selectable Auth | | 2.1.9 | 2024-05-17 | [38301](https://github.com/airbytehq/airbyte/pull/38301) | Fix data inaccuracies when `wish_bid` is requested | | 2.1.8 | 2024-05-07 | [37771](https://github.com/airbytehq/airbyte/pull/37771) | Handle errors without API error codes/messages | | 2.1.7 | 2024-04-24 | [36634](https://github.com/airbytehq/airbyte/pull/36634) | Update to CDK 0.80.0 | From fca9c4c6a1be3cadc8106768085514c24feb07fa Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Wed, 29 May 2024 04:55:36 -0600 Subject: [PATCH 09/20] fixes --- .../source_facebook_marketing/source.py | 13 +++++++++---- .../source_facebook_marketing/spec.py | 8 ++++++++ .../unit_tests/integration/config.py | 2 +- .../unit_tests/test_config_migrations.py | 1 + .../account_id_to_array/test_old_config.json | 15 +-------------- .../include_deleted_true/test_old_config.json | 16 +--------------- .../test_new_access_token_config.json | 1 + .../test_new_client_config.json | 1 + .../test_old_access_token_config.json | 15 +-------------- .../test_old_client_config.json | 16 +--------------- 10 files changed, 25 insertions(+), 63 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 95332c4ca1726..c45482e1be830 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -129,10 +129,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: config.start_date = validate_start_date(config.start_date) config.end_date = validate_end_date(config.start_date, config.end_date) - if config.credentials.auth_type == "Client": - api = API(access_token=config.credentials.access_token, page_size=config.page_size) - if config.credentials.auth_type == "Service": - api = API(access_token=config.credentials.access_token, page_size=config.page_size) + api = API(access_token=config.credentials.access_token, page_size=config.page_size) # if start_date not specified then set default start_date for report streams to 2 years ago report_start_date = config.start_date or pendulum.now().add(years=-2) @@ -254,6 +251,10 @@ def spec(self, *args, **kwargs) -> ConnectorSpecification: "access_token": { "type": "string", "path_in_connector_config": ["credentials", "access_token"], + }, + "auth_type": { + "type": "string", + "path_in_connector_config": ["credentials", "auth_type"], } }, }, @@ -276,6 +277,10 @@ def spec(self, *args, **kwargs) -> ConnectorSpecification: "type": "string", "path_in_connector_config": ["credentials", "client_secret"], }, + "auth_type": { + "type": "string", + "path_in_connector_config": ["credentials", "auth_type"], + } }, }, ), diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index ad4d800a1c2ab..c05923c870488 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -44,6 +44,14 @@ class Config(OneOfOptionConfig): description="Client Secret for the Facebook Marketing API", airbyte_secret=True, ) + access_token: Optional[str] = Field( + title="Access Token", + description="The value of the generated access token. " + 'From your App’s Dashboard, click on "Marketing API" then "Tools". ' + 'Select permissions ads_management, ads_read, read_insights, business_management. Then click on "Get token". ' + 'See the docs for more information.', + airbyte_secret=True, + ) class ServiceAccountCredentials(BaseModel): diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py index 5edc08332f1b1..f6fb44b55c18a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py @@ -27,7 +27,7 @@ def __init__(self) -> None: "account_ids": [ACCOUNT_ID], "credentials": { "auth_type": "Service", - "ACCESS_TOKEN": ACCESS_TOKEN, + "access_token": ACCESS_TOKEN, }, "start_date": START_DATE, "end_date": END_DATE, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py index 8a99bc97995c7..ffc87e82abab5 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py @@ -179,6 +179,7 @@ def revert_migration(config_path: str) -> None: with open(config_path, "r") as test_config: config = json.load(test_config) credentials = config.pop("credentials",{}) + credentials.pop("auth_type", None) with open(config_path, "w") as updated_config: config = json.dumps({**config, **credentials}) updated_config.write(config) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json index a04560eb77103..526c0c0caa7ef 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json @@ -1,14 +1 @@ -{ - "start_date": "2021-02-08T00:00:00Z", - "end_date": "2021-02-15T00:00:00Z", - "custom_insights": [ - { - "name": "custom_insight_stream", - "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], - "breakdowns": ["gender"], - "action_breakdowns": [] - } - ], - "account_id": "01234567890", - "access_token": "access_token" -} +{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "account_id": "01234567890", "access_token": "access_token"} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json index 0cf00a31758d4..52234b6754295 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json @@ -1,15 +1 @@ -{ - "start_date": "2021-02-08T00:00:00Z", - "end_date": "2021-02-15T00:00:00Z", - "custom_insights": [ - { - "name": "custom_insight_stream", - "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], - "breakdowns": ["gender"], - "action_breakdowns": [] - } - ], - "include_deleted": true, - "account_ids": ["01234567890"], - "access_token": "access_token" -} +{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "include_deleted": true, "account_ids": ["01234567890"], "access_token": "access_token"} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_access_token_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_access_token_config.json index 6ebf309b081ba..f735cac033a0a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_access_token_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_access_token_config.json @@ -11,6 +11,7 @@ ], "account_ids": ["01234567890"], "credentials": { + "auth_type": "Service", "access_token": "access_token" } } diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_client_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_client_config.json index a8ebfdf994b93..6a30bdb8b82a3 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_client_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_new_client_config.json @@ -11,6 +11,7 @@ ], "account_ids": ["01234567890"], "credentials": { + "auth_type": "Client", "client_id": "client_id", "client_secret": "client_secret" } diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json index a04560eb77103..437fff9ec7de9 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json @@ -1,14 +1 @@ -{ - "start_date": "2021-02-08T00:00:00Z", - "end_date": "2021-02-15T00:00:00Z", - "custom_insights": [ - { - "name": "custom_insight_stream", - "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], - "breakdowns": ["gender"], - "action_breakdowns": [] - } - ], - "account_id": "01234567890", - "access_token": "access_token" -} +{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "account_id": "01234567890", "auth_type": "Service", "access_token": "access_token"} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json index eee9ac482f9eb..b7e62594b8775 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json @@ -1,15 +1 @@ -{ - "start_date": "2021-02-08T00:00:00Z", - "end_date": "2021-02-15T00:00:00Z", - "custom_insights": [ - { - "name": "custom_insight_stream", - "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], - "breakdowns": ["gender"], - "action_breakdowns": [] - } - ], - "account_id": "01234567890", - "client_id": "client_id", - "client_secret": "client_secret" -} +{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "account_id": "01234567890", "auth_type": "Client", "client_id": "client_id", "client_secret": "client_secret"} \ No newline at end of file From 9f1b19f0eb697cc33499d5eba67b867eae2a3409 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Thu, 30 May 2024 08:40:43 -0600 Subject: [PATCH 10/20] Supporting credentials at both locations --- .../tests/fixtures/cloud_registry.json | 37 +++--------------- .../tests/fixtures/oss_registry.json | 39 ++++--------------- .../config_migrations.py | 34 ++++++++-------- .../source_facebook_marketing/source.py | 10 ++++- .../source_facebook_marketing/spec.py | 24 ++++++++++++ .../unit_tests/integration/config.py | 1 + .../unit_tests/test_config_migrations.py | 16 ++------ .../unit_tests/test_source.py | 1 + 8 files changed, 67 insertions(+), 95 deletions(-) diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/cloud_registry.json b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/cloud_registry.json index c95e9544e86ef..dc87958945887 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/cloud_registry.json +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/cloud_registry.json @@ -8778,37 +8778,12 @@ "examples": ["2017-01-26T00:00:00Z"], "type": "string" }, - "credentials": { - "title": "authentication credentials", - "description": "", - "type": "object", - "properties": { - "access_token": { - "title": "Access Token", - "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", - "order": 3, - "airbyte_secret": true, - "type": "string" - }, - "client_id": { - "title": "Client Id", - "description": "The Client Id for your OAuth app", - "airbyte_secret": true, - "type": "string" - }, - "client_secret": { - "title": "Client Secret", - "description": "The Client Secret for your OAuth app", - "airbyte_secret": true, - "type": "string" - }, - "auth_type": { - "title": "Auth Type", - "description": "The type of authentication to use. If you are using an OAuth app, select 'oauth'. If you are using an access token, select 'access_token'.", - "enum": ["Service", "Client"], - "type": "string" - } - } + "access_token": { + "title": "Access Token", + "description": "The value of the generated access token. From your App’s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", + "order": 3, + "airbyte_secret": true, + "type": "string" }, "include_deleted": { "title": "Include Deleted Campaigns, Ads, and AdSets", diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json index 0460940b6d45d..841017aad4faa 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json @@ -13214,37 +13214,12 @@ "examples": ["2017-01-26T00:00:00Z"], "type": "string" }, - "credentials": { - "title": "authentication credentials", - "description": "", - "type": "object", - "properties": { - "access_token": { - "title": "Access Token", - "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", - "order": 3, - "airbyte_secret": true, - "type": "string" - }, - "client_id": { - "title": "Client Id", - "description": "The Client Id for your OAuth app", - "airbyte_secret": true, - "type": "string" - }, - "client_secret": { - "title": "Client Secret", - "description": "The Client Secret for your OAuth app", - "airbyte_secret": true, - "type": "string" - }, - "auth_type": { - "title": "Auth Type", - "description": "The type of authentication to use. If you are using an OAuth app, select 'oauth'. If you are using an access token, select 'access_token'.", - "enum": ["Service", "Client"], - "type": "string" - } - } + "access_token": { + "title": "Access Token", + "description": "The value of the generated access token. From your App’s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", + "order": 3, + "airbyte_secret": true, + "type": "string" }, "include_deleted": { "title": "Include Deleted Campaigns, Ads, and AdSets", @@ -30484,4 +30459,4 @@ "releaseStage": "alpha" } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py index be0d7d71e7fd4..2f688f27a5ba0 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py @@ -131,10 +131,9 @@ class MigrateSecretsPathInConnector: Starting from `2.2.0`, the `client_id`, `client_secret` and `access_token` will be placed at `credentials` path. """ - message_repository: MessageRepository = InMemoryMessageRepository() @classmethod - def should_migrate(cls, config: Mapping[str, Any]) -> bool: + def _should_migrate(cls, config: Mapping[str, Any]) -> bool: """ This method determines whether the config should be migrated to nest existing fields at credentials. It is assumed if credentials does not exist on configuration, `client_id`, `client_secret` and `access_token` exists on root path. @@ -160,41 +159,40 @@ def migrate(cls, args: List[str], source: Source) -> None: # read the existing config config = source.read_config(config_path) # migration check - if cls.should_migrate(config): - cls.emit_control_message( - cls.modify_and_save(config_path, source, config), + if cls._should_migrate(config): + cls._emit_control_message( + cls._modify_and_save(config_path, source, config), ) @classmethod - def transform(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: + def _transform(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: # transform the config if "credentials" not in config: - config["credentials"] = {} + config["credentials"] = { + "auth_type": "Service", + } if "access_token" in config: - config["credentials"]["auth_type"] = "Service" - config["credentials"]["access_token"] = config.pop("access_token") + config["credentials"]["access_token"] = config.get("access_token") if "client_id" in config: config["credentials"]["auth_type"] = "Client" - config["credentials"]["client_id"] = config.pop("client_id") + config["credentials"]["client_id"] = config.get("client_id") if "client_secret" in config: config["credentials"]["auth_type"] = "Client" - config["credentials"]["client_secret"] = config.pop("client_secret") + config["credentials"]["client_secret"] = config.get("client_secret") # return transformed config return config @classmethod - def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: + def _modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: # modify the config - migrated_config = cls.transform(config) + migrated_config = cls._transform(config) # save the config source.write_config(migrated_config, config_path) # return modified config return migrated_config @classmethod - def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: # add the Airbyte Control Message to message repo - cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) - # emit the Airbyte Control Message from message queue to stdout - for message in cls.message_repository._message_queue: - print(message.json(exclude_unset=True)) + print(create_connector_config_control_message(migrated_config).json(exclude_unset=True)) + diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index c45482e1be830..c90ff97168f5e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -95,7 +95,10 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> if config.start_date and config.end_date < config.start_date: return False, "End date must be equal or after start date." - api = API(access_token=config.credentials.access_token, page_size=config.page_size) + if config.credentials is not None: + api = API(access_token=config.credentials.access_token, page_size=config.page_size) + else: + api = API(access_token=config.access_token, page_size=config.page_size) for account_id in config.account_ids: # Get Ad Account to check creds @@ -129,7 +132,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: config.start_date = validate_start_date(config.start_date) config.end_date = validate_end_date(config.start_date, config.end_date) - api = API(access_token=config.credentials.access_token, page_size=config.page_size) + if config.credentials is not None: + api = API(access_token=config.credentials.access_token, page_size=config.page_size) + else: + api = API(access_token=config.access_token, page_size=config.page_size) # if start_date not specified then set default start_date for report streams to 2 years ago report_start_date = config.start_date or pendulum.now().add(years=-2) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index c05923c870488..932c6375a71cd 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -181,6 +181,18 @@ class Config: min_items=1, ) + access_token: str = Field( + title="Access Token", + order=1, + description=( + "The value of the generated access token. " + 'From your App’s Dashboard, click on "Marketing API" then "Tools". ' + 'Select permissions ads_management, ads_read, read_insights, business_management. Then click on "Get token". ' + 'See the docs for more information.' + ), + airbyte_secret=True, + ) + credentials: Union[OAuthCredentials, ServiceAccountCredentials] = Field( title="Authentication", description="Credentials for connecting to the Facebook Marketing API", @@ -292,3 +304,15 @@ class Config: default=True, airbyte_hidden=True, ) + + client_id: Optional[str] = Field( + description="The Client Id for your OAuth app", + airbyte_secret=True, + airbyte_hidden=True, + ) + + client_secret: Optional[str] = Field( + description="The Client Secret for your OAuth app", + airbyte_secret=True, + airbyte_hidden=True, + ) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py index f6fb44b55c18a..1e76b6403a501 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py @@ -25,6 +25,7 @@ class ConfigBuilder: def __init__(self) -> None: self._config: MutableMapping[str, Any] = { "account_ids": [ACCOUNT_ID], + "access_token": ACCESS_TOKEN, "credentials": { "auth_type": "Service", "access_token": ACCESS_TOKEN, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py index ffc87e82abab5..54bb48f02a1b9 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py @@ -197,13 +197,9 @@ def test_migrate_access_token_config(self): credentials = test_migrated_config["credentials"] assert "access_token" in credentials # check the migration should be skipped, once already done - assert not migration_instance.should_migrate(test_migrated_config) + assert not migration_instance._should_migrate(test_migrated_config) # load the old custom reports VS migrated assert original_config["access_token"] == credentials["access_token"] - # test CONTROL MESSAGE was emitted - control_msg = migration_instance.message_repository._message_queue[0] - assert control_msg.type == Type.CONTROL - assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG # revert the test_config to the starting point self.revert_migration(self.OLD_TEST_CONFIG_PATH_ACCESS_TOKEN) @@ -221,23 +217,19 @@ def test_migrate_client_config(self): assert "client_id" in credentials assert "client_secret" in credentials # check the migration should be skipped, once already done - assert not migration_instance.should_migrate(test_migrated_config) + assert not migration_instance._should_migrate(test_migrated_config) # load the old custom reports VS migrated assert original_config["client_id"] == credentials["client_id"] assert original_config["client_secret"] == credentials["client_secret"] - # test CONTROL MESSAGE was emitted - control_msg = migration_instance.message_repository._message_queue[0] - assert control_msg.type == Type.CONTROL - assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG # revert the test_config to the starting point self.revert_migration(self.OLD_TEST_CONFIG_PATH_CLIENT) def test_should_not_migrate_new_client_config(self): new_config = load_config(self.NEW_TEST_CONFIG_PATH_CLIENT) migration_instance = MigrateSecretsPathInConnector() - assert not migration_instance.should_migrate(new_config) + assert not migration_instance._should_migrate(new_config) def test_should_not_migrate_new_access_token_config(self): new_config = load_config(self.NEW_TEST_CONFIG_PATH_ACCESS_TOKEN) migration_instance = MigrateSecretsPathInConnector() - assert not migration_instance.should_migrate(new_config) + assert not migration_instance._should_migrate(new_config) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py index e264760ea701b..70d7653152943 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py @@ -28,6 +28,7 @@ def config_fixture(requests_mock): config = { "account_ids": ["123"], + "access_token": "ACCESS_TOKEN", "credentials": { "auth_type": "Service", "access_token": "ACCESS_TOKEN", From ff417ddb529d470e014073bb095b1687ccd9b201 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Fri, 31 May 2024 06:59:02 -0600 Subject: [PATCH 11/20] fixes spec --- .../tests/fixtures/oss_registry.json | 2 +- .../integration_tests/spec.json | 23 ++++++++++++++++++- .../config_migrations.py | 13 ++++------- .../source_facebook_marketing/source.py | 6 ++--- .../source_facebook_marketing/spec.py | 4 ++-- .../unit_tests/integration/request_builder.py | 2 +- .../test_ads_insights_action_product_id.py | 2 +- .../unit_tests/integration/test_videos.py | 2 +- 8 files changed, 35 insertions(+), 19 deletions(-) diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json index 841017aad4faa..a1271847cddbe 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json @@ -30459,4 +30459,4 @@ "releaseStage": "alpha" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index de74a2ea77405..b97c37861ddbd 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -19,6 +19,13 @@ }, "uniqueItems": true }, + "access_token": { + "title": "Access Token", + "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", + "order": 1, + "airbyte_secret": true, + "type": "string" + }, "credentials": { "title": "authentication credentials", "description": "", @@ -473,9 +480,23 @@ "default": true, "airbyte_hidden": true, "type": "boolean" + }, + "client_id": { + "title": "Client Id", + "description": "The Client Id for your OAuth app", + "airbyte_secret": true, + "airbyte_hidden": true, + "type": "string" + }, + "client_secret": { + "title": "Client Secret", + "description": "The Client Secret for your OAuth app", + "airbyte_secret": true, + "airbyte_hidden": true, + "type": "string" } }, - "required": ["account_ids", "credentials"] + "required": ["account_ids"] }, "supportsIncremental": true, "supported_destination_sync_modes": ["append"], diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py index 2f688f27a5ba0..67d54a939428a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py @@ -131,7 +131,6 @@ class MigrateSecretsPathInConnector: Starting from `2.2.0`, the `client_id`, `client_secret` and `access_token` will be placed at `credentials` path. """ - @classmethod def _should_migrate(cls, config: Mapping[str, Any]) -> bool: """ @@ -141,10 +140,7 @@ def _should_migrate(cls, config: Mapping[str, Any]) -> bool: > True, if the migration is necessary > False, otherwise. """ - credentials = config.get("credentials", None) - return credentials is None or ( - "access_token" not in credentials and ("client_id" not in credentials or "client_secret" not in credentials) - ) + return "access_token" in config or "client_id" in config or "client_secret" in config @classmethod def migrate(cls, args: List[str], source: Source) -> None: @@ -172,13 +168,13 @@ def _transform(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: "auth_type": "Service", } if "access_token" in config: - config["credentials"]["access_token"] = config.get("access_token") + config["credentials"]["access_token"] = config.pop("access_token") if "client_id" in config: config["credentials"]["auth_type"] = "Client" - config["credentials"]["client_id"] = config.get("client_id") + config["credentials"]["client_id"] = config.pop("client_id") if "client_secret" in config: config["credentials"]["auth_type"] = "Client" - config["credentials"]["client_secret"] = config.get("client_secret") + config["credentials"]["client_secret"] = config.pop("client_secret") # return transformed config return config @@ -195,4 +191,3 @@ def _modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: # add the Airbyte Control Message to message repo print(create_connector_config_control_message(migrated_config).json(exclude_unset=True)) - diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index c90ff97168f5e..e6c4b93e2f0d8 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -133,7 +133,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: config.end_date = validate_end_date(config.start_date, config.end_date) if config.credentials is not None: - api = API(access_token=config.credentials.access_token, page_size=config.page_size) + api = API(access_token=config.credentials.access_token, page_size=config.page_size) else: api = API(access_token=config.access_token, page_size=config.page_size) @@ -261,7 +261,7 @@ def spec(self, *args, **kwargs) -> ConnectorSpecification: "auth_type": { "type": "string", "path_in_connector_config": ["credentials", "auth_type"], - } + }, }, }, complete_oauth_server_input_specification={ @@ -286,7 +286,7 @@ def spec(self, *args, **kwargs) -> ConnectorSpecification: "auth_type": { "type": "string", "path_in_connector_config": ["credentials", "auth_type"], - } + }, }, }, ), diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index 932c6375a71cd..ea9d96e33428a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -181,7 +181,7 @@ class Config: min_items=1, ) - access_token: str = Field( + access_token: Optional[str] = Field( title="Access Token", order=1, description=( @@ -193,7 +193,7 @@ class Config: airbyte_secret=True, ) - credentials: Union[OAuthCredentials, ServiceAccountCredentials] = Field( + credentials: Optional[Union[OAuthCredentials, ServiceAccountCredentials]] = Field( title="Authentication", description="Credentials for connecting to the Facebook Marketing API", discriminator="auth_type", diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py index c6ed5efddbe98..ed523146085b5 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/request_builder.py @@ -9,7 +9,7 @@ from airbyte_cdk.test.mock_http.request import HttpRequest -from .config import ACCOUNT_ID, ACCESS_TOKEN +from .config import ACCESS_TOKEN, ACCOUNT_ID def get_account_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py index 151635f72cb04..f3d0426a273ed 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py @@ -25,7 +25,7 @@ from airbyte_protocol.models import AirbyteStateMessage, StreamDescriptor, SyncMode from source_facebook_marketing.streams.async_job import Status -from .config import ACCOUNT_ID, DATE_FORMAT, END_DATE, NOW, ACCESS_TOKEN, START_DATE, ConfigBuilder +from .config import ACCESS_TOKEN, ACCOUNT_ID, DATE_FORMAT, END_DATE, NOW, START_DATE, ConfigBuilder from .pagination import NEXT_PAGE_TOKEN, FacebookMarketingPaginationStrategy from .request_builder import RequestBuilder, get_account_request from .response_builder import build_response, error_reduce_amount_of_data_response, get_account_response diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py index d2e425bee2f3a..ee814fd201f0b 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py @@ -20,7 +20,7 @@ from airbyte_cdk.test.state_builder import StateBuilder from airbyte_protocol.models import AirbyteStateMessage, SyncMode -from .config import ACCOUNT_ID, NOW, ACCESS_TOKEN, ConfigBuilder +from .config import ACCESS_TOKEN, ACCOUNT_ID, NOW, ConfigBuilder from .pagination import NEXT_PAGE_TOKEN, FacebookMarketingPaginationStrategy from .request_builder import RequestBuilder, get_account_request from .response_builder import error_reduce_amount_of_data_response, get_account_response From 985d552f34bb699cb993d7496d819ba0e35e4b20 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Fri, 31 May 2024 07:26:06 -0600 Subject: [PATCH 12/20] re-format test jsons --- .../account_id_to_array/test_old_config.json | 15 ++++++++++++++- .../include_deleted_true/test_old_config.json | 16 +++++++++++++++- .../test_old_access_token_config.json | 16 +++++++++++++++- .../test_old_client_config.json | 17 ++++++++++++++++- 4 files changed, 60 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json index 526c0c0caa7ef..a04560eb77103 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/account_id_to_array/test_old_config.json @@ -1 +1,14 @@ -{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "account_id": "01234567890", "access_token": "access_token"} \ No newline at end of file +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_id": "01234567890", + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json index 52234b6754295..0cf00a31758d4 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/include_deleted_to_status_filters/include_deleted_true/test_old_config.json @@ -1 +1,15 @@ -{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "include_deleted": true, "account_ids": ["01234567890"], "access_token": "access_token"} \ No newline at end of file +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "include_deleted": true, + "account_ids": ["01234567890"], + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json index 437fff9ec7de9..b499ed5ab91a8 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_access_token_config.json @@ -1 +1,15 @@ -{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "account_id": "01234567890", "auth_type": "Service", "access_token": "access_token"} \ No newline at end of file +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_id": "01234567890", + "auth_type": "Service", + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json index b7e62594b8775..e8434efbec9b7 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/secrets_to_credentials/test_old_client_config.json @@ -1 +1,16 @@ -{"start_date": "2021-02-08T00:00:00Z", "end_date": "2021-02-15T00:00:00Z", "custom_insights": [{"name": "custom_insight_stream", "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], "breakdowns": ["gender"], "action_breakdowns": []}], "account_id": "01234567890", "auth_type": "Client", "client_id": "client_id", "client_secret": "client_secret"} \ No newline at end of file +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_id": "01234567890", + "auth_type": "Client", + "client_id": "client_id", + "client_secret": "client_secret" +} From 53806491eb335f40452c032c0f739f2e4903ad4e Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Fri, 31 May 2024 12:24:03 -0600 Subject: [PATCH 13/20] affinf auth type to spec --- .../source-facebook-marketing/integration_tests/spec.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index b97c37861ddbd..371da7165bc7b 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -494,6 +494,13 @@ "airbyte_secret": true, "airbyte_hidden": true, "type": "string" + }, + "auth_type": { + "title": "Auth Type", + "default": "Server", + "const": "Server", + "enum": ["Server", "Client"], + "type": "string" } }, "required": ["account_ids"] From c6b0fed8d151105a30293ede2bb5263c9fec8f39 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Mon, 3 Jun 2024 03:22:00 -0600 Subject: [PATCH 14/20] removing auth type from root --- .../source-facebook-marketing/integration_tests/spec.json | 7 ------- 1 file changed, 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index 371da7165bc7b..b97c37861ddbd 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -494,13 +494,6 @@ "airbyte_secret": true, "airbyte_hidden": true, "type": "string" - }, - "auth_type": { - "title": "Auth Type", - "default": "Server", - "const": "Server", - "enum": ["Server", "Client"], - "type": "string" } }, "required": ["account_ids"] From 7e9451fdcdab571f99e846ea5c3d5e3dd9a52151 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Mon, 3 Jun 2024 06:36:52 -0600 Subject: [PATCH 15/20] rm spec.json --- .../integration_tests/spec.json | 34 +------------------ 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index b97c37861ddbd..328659d53acce 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -26,38 +26,6 @@ "airbyte_secret": true, "type": "string" }, - "credentials": { - "title": "authentication credentials", - "description": "", - "type": "object", - "properties": { - "access_token": { - "title": "Access Token", - "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", - "order": 1, - "airbyte_secret": true, - "type": "string" - }, - "client_id": { - "title": "Client Id", - "description": "The Client Id for your OAuth app", - "airbyte_secret": true, - "type": "string" - }, - "client_secret": { - "title": "Client Secret", - "description": "The Client Secret for your OAuth app", - "airbyte_secret": true, - "type": "string" - }, - "auth_type": { - "title": "Auth Type", - "description": "The type of authentication to use. If you are using an OAuth app, select 'oauth'. If you are using an access token, select 'access_token'.", - "enum": ["Service", "Client"], - "type": "string" - } - } - }, "start_date": { "title": "Start Date", "description": "The date from which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. If not set then all data will be replicated for usual streams and only last 2 years for insight streams.", @@ -496,7 +464,7 @@ "type": "string" } }, - "required": ["account_ids"] + "required": ["account_ids", "access_token"] }, "supportsIncremental": true, "supported_destination_sync_modes": ["append"], From eb2498ba7e2a007010a9395aebbabc26e5d0b94b Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 4 Jun 2024 03:51:18 -0600 Subject: [PATCH 16/20] version bump --- .../connectors/source-facebook-marketing/metadata.yaml | 2 +- .../connectors/source-facebook-marketing/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml index 432dda39f892b..af996f249492a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c - dockerImageTag: 3.1.0 + dockerImageTag: 3.2.0 dockerRepository: airbyte/source-facebook-marketing documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing githubIssueLabel: source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml b/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml index 68df9dfb0b75e..fb99288fdc3ca 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml +++ b/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "3.1.0" +version = "3.2.0" name = "source-facebook-marketing" description = "Source implementation for Facebook Marketing." authors = [ "Airbyte ",] From 70216793868660b009e9cf58862cb373533702ec Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 4 Jun 2024 13:38:53 -0600 Subject: [PATCH 17/20] adding back spec --- .../integration_tests/spec.json | 95 ++++++++++++++++++- 1 file changed, 91 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index 4566a1a627e7e..aab04fc471555 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -26,6 +26,83 @@ "airbyte_secret": true, "type": "string" }, + "credentials": { + "title": "Authentication", + "description": "Credentials for connecting to the Facebook Marketing API", + "type": "object", + "discriminator": { + "propertyName": "auth_type", + "mapping": { + "Client": "#/definitions/OAuthCredentials", + "Service": "#/definitions/ServiceAccountCredentials" + } + }, + "oneOf": [ + { + "title": "Authenticate via Facebook Marketing (Oauth)", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "default": "Client", + "const": "Client", + "enum": [ + "Client" + ], + "type": "string" + }, + "client_id": { + "title": "Client ID", + "description": "Client ID for the Facebook Marketing API", + "airbyte_secret": true, + "type": "string" + }, + "client_secret": { + "title": "Client Secret", + "description": "Client Secret for the Facebook Marketing API", + "airbyte_secret": true, + "type": "string" + }, + "access_token": { + "title": "Access Token", + "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", + "airbyte_secret": true, + "type": "string" + } + }, + "required": [ + "client_id", + "client_secret", + "auth_type" + ] + }, + { + "title": "Service Account Key Authentication", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "default": "Service", + "const": "Service", + "enum": [ + "Service" + ], + "type": "string" + }, + "access_token": { + "title": "Access Token", + "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", + "airbyte_secret": true, + "type": "string" + } + }, + "required": [ + "access_token", + "auth_type" + ] + } + ] + }, "start_date": { "title": "Start Date", "description": "The date from which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. If not set then all data will be replicated for usual streams and only last 2 years for insight streams.", @@ -462,19 +539,25 @@ "type": "string" } }, - "required": ["account_ids", "access_token"] + "required": ["account_ids"] }, "supportsIncremental": true, "supported_destination_sync_modes": ["append"], "advanced_auth": { "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials","auth_type"], + "predicate_value": "Client", "oauth_config_specification": { "complete_oauth_output_specification": { "type": "object", "properties": { "access_token": { "type": "string", - "path_in_connector_config": ["access_token"] + "path_in_connector_config": ["credentials","access_token"] + }, + "auth_type": { + "type": "string", + "path_in_connector_config": ["credentials","auth_type"] } } }, @@ -495,11 +578,15 @@ "properties": { "client_id": { "type": "string", - "path_in_connector_config": ["client_id"] + "path_in_connector_config": ["credentials","client_id"] }, "client_secret": { "type": "string", - "path_in_connector_config": ["client_secret"] + "path_in_connector_config": ["credentials","client_secret"] + }, + "auth_type": { + "type": "string", + "path_in_connector_config": ["credentials","auth_type"] } } } From 8fdca536e361c3dcc8c7f175c4829225a91f38cc Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 4 Jun 2024 13:52:58 -0600 Subject: [PATCH 18/20] fix format --- .../integration_tests/spec.json | 31 ++++++------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index aab04fc471555..b8685c51159ad 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -46,9 +46,7 @@ "title": "Auth Type", "default": "Client", "const": "Client", - "enum": [ - "Client" - ], + "enum": ["Client"], "type": "string" }, "client_id": { @@ -70,11 +68,7 @@ "type": "string" } }, - "required": [ - "client_id", - "client_secret", - "auth_type" - ] + "required": ["client_id", "client_secret", "auth_type"] }, { "title": "Service Account Key Authentication", @@ -84,9 +78,7 @@ "title": "Auth Type", "default": "Service", "const": "Service", - "enum": [ - "Service" - ], + "enum": ["Service"], "type": "string" }, "access_token": { @@ -96,10 +88,7 @@ "type": "string" } }, - "required": [ - "access_token", - "auth_type" - ] + "required": ["access_token", "auth_type"] } ] }, @@ -545,7 +534,7 @@ "supported_destination_sync_modes": ["append"], "advanced_auth": { "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials","auth_type"], + "predicate_key": ["credentials", "auth_type"], "predicate_value": "Client", "oauth_config_specification": { "complete_oauth_output_specification": { @@ -553,11 +542,11 @@ "properties": { "access_token": { "type": "string", - "path_in_connector_config": ["credentials","access_token"] + "path_in_connector_config": ["credentials", "access_token"] }, "auth_type": { "type": "string", - "path_in_connector_config": ["credentials","auth_type"] + "path_in_connector_config": ["credentials", "auth_type"] } } }, @@ -578,15 +567,15 @@ "properties": { "client_id": { "type": "string", - "path_in_connector_config": ["credentials","client_id"] + "path_in_connector_config": ["credentials", "client_id"] }, "client_secret": { "type": "string", - "path_in_connector_config": ["credentials","client_secret"] + "path_in_connector_config": ["credentials", "client_secret"] }, "auth_type": { "type": "string", - "path_in_connector_config": ["credentials","auth_type"] + "path_in_connector_config": ["credentials", "auth_type"] } } } From 543d5ea079a27e0e50f6c3109b75577126de996d Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Wed, 5 Jun 2024 20:05:54 -0600 Subject: [PATCH 19/20] merge from master and remove auth_type --- .../source_facebook_marketing/source.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index e6c4b93e2f0d8..811d3c41b84ac 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -258,10 +258,6 @@ def spec(self, *args, **kwargs) -> ConnectorSpecification: "type": "string", "path_in_connector_config": ["credentials", "access_token"], }, - "auth_type": { - "type": "string", - "path_in_connector_config": ["credentials", "auth_type"], - }, }, }, complete_oauth_server_input_specification={ @@ -283,10 +279,6 @@ def spec(self, *args, **kwargs) -> ConnectorSpecification: "type": "string", "path_in_connector_config": ["credentials", "client_secret"], }, - "auth_type": { - "type": "string", - "path_in_connector_config": ["credentials", "auth_type"], - }, }, }, ), From ce319201fbbef31bcbc8ec3401057d0e643cfb20 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 6 Jun 2024 13:45:27 +0300 Subject: [PATCH 20/20] fixed failed CAT test --- .../source-facebook-marketing/acceptance-test-config.yml | 5 +++-- .../source-facebook-marketing/integration_tests/spec.json | 8 -------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml index dae55df93f18f..c7cdf62e8bb7e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml @@ -6,9 +6,10 @@ acceptance_tests: spec: tests: - spec_path: "integration_tests/spec.json" + # the source now supports 2 auth types, thus we should skip the check, + # this change is intentional backward_compatibility_tests_config: - disable_for_version: "1.2.2" - previous_connector_version: "1.2.1" + disable_for_version: "3.1.0" connection: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index b8685c51159ad..7e56eab74cf6d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -543,10 +543,6 @@ "access_token": { "type": "string", "path_in_connector_config": ["credentials", "access_token"] - }, - "auth_type": { - "type": "string", - "path_in_connector_config": ["credentials", "auth_type"] } } }, @@ -572,10 +568,6 @@ "client_secret": { "type": "string", "path_in_connector_config": ["credentials", "client_secret"] - }, - "auth_type": { - "type": "string", - "path_in_connector_config": ["credentials", "auth_type"] } } }