diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 0ebafb5b8240..332b1c0fe68d 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -50,6 +50,7 @@ jobs: BIGQUERY_INTEGRATION_TEST_CREDS: ${{ secrets.BIGQUERY_INTEGRATION_TEST_CREDS }} BRAINTREE_TEST_CREDS: ${{ secrets.BRAINTREE_TEST_CREDS }} DRIFT_INTEGRATION_TEST_CREDS: ${{ secrets.DRIFT_INTEGRATION_TEST_CREDS }} + FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS }} FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS }} FRESHDESK_TEST_CREDS: ${{ secrets.FRESHDESK_TEST_CREDS }} GH_INTEGRATION_TEST_CREDS: ${{ secrets.GH_INTEGRATION_TEST_CREDS }} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/74d47f79-8d01-44ac-9755-f5eb0d7caacb.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/74d47f79-8d01-44ac-9755-f5eb0d7caacb.json deleted file mode 100644 index 95f3c31e81a4..000000000000 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/74d47f79-8d01-44ac-9755-f5eb0d7caacb.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "sourceDefinitionId": "74d47f79-8d01-44ac-9755-f5eb0d7caacb", - "name": "Facebook Marketing APIs", - "dockerRepository": "airbyte/source-facebook-marketing-api-singer", - "dockerImageTag": "0.1.5", - "documentationUrl": "https://hub.docker.com/r/airbyte/source-facebook-marketing-api-singer" -} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json new file mode 100644 index 000000000000..b903398820e0 --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "e7778cfc-e97c-4458-9ecb-b4f2bba8946c", + "name": "Facebook Marketing", + "dockerRepository": "airbyte/source-facebook-marketing", + "dockerImageTag": "0.1.1", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-facebook-marketing" +} diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 4f581f69a046..6d8a87ab90ec 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -73,11 +73,11 @@ dockerRepository: airbyte/source-googleanalytics-singer dockerImageTag: 0.1.5 documentationUrl: https://hub.docker.com/r/airbyte/source-googleanalytics-singer -- sourceDefinitionId: 74d47f79-8d01-44ac-9755-f5eb0d7caacb - name: Facebook Marketing APIs - dockerRepository: airbyte/source-facebook-marketing-api-singer - dockerImageTag: 0.1.5 - documentationUrl: https://hub.docker.com/r/airbyte/source-facebook-marketing-api-singer +- sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c + name: Facebook Marketing + dockerRepository: airbyte/source-facebook-marketing + dockerImageTag: 0.1.1 + documentationUrl: https://hub.docker.com/r/airbyte/source-facebook-marketing - sourceDefinitionId: 57eb1576-8f52-463d-beb6-2e107cdf571d name: Hubspot dockerRepository: airbyte/source-hubspot-singer diff --git a/airbyte-integrations/bases/base-python/Dockerfile b/airbyte-integrations/bases/base-python/Dockerfile index 1fece0428767..bf20062f074d 100644 --- a/airbyte-integrations/bases/base-python/Dockerfile +++ b/airbyte-integrations/bases/base-python/Dockerfile @@ -4,6 +4,7 @@ COPY --from=airbyte/integration-base:dev /airbyte /airbyte WORKDIR /airbyte/base_python_code COPY base_python ./base_python COPY setup.py ./ +RUN pip install pip==20.2 RUN pip install . ENV AIRBYTE_SPEC_CMD "base-python spec" diff --git a/airbyte-integrations/bases/base-python/base_python/client.py b/airbyte-integrations/bases/base-python/base_python/client.py index dfa2640f41fe..49c3233a9273 100644 --- a/airbyte-integrations/bases/base-python/base_python/client.py +++ b/airbyte-integrations/bases/base-python/base_python/client.py @@ -24,12 +24,15 @@ import inspect import json +import os import pkgutil from abc import ABC, abstractmethod from datetime import datetime -from typing import Dict, Generator, Tuple +from typing import Dict, Generator, List, Tuple +import pkg_resources from airbyte_protocol import AirbyteRecordMessage, AirbyteStream +from jsonschema import RefResolver def package_name_from_class(cls: object) -> str: @@ -38,6 +41,65 @@ def package_name_from_class(cls: object) -> str: return module.__name__.split(".")[0] +class JsonSchemaResolver: + """Helper class to expand $ref items in json schema""" + + def __init__(self, shared_schemas_path: str): + self._shared_refs = self._load_shared_schema_refs(shared_schemas_path) + + @staticmethod + def _load_shared_schema_refs(path: str): + shared_file_names = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))] + + shared_schema_refs = {} + for shared_file in shared_file_names: + with open(os.path.join(path, shared_file)) as data_file: + shared_schema_refs[shared_file] = json.load(data_file) + + return shared_schema_refs + + def _resolve_schema_references(self, schema: dict, resolver: RefResolver) -> dict: + if "$ref" in schema: + reference_path = schema.pop("$ref", None) + resolved = resolver.resolve(reference_path)[1] + schema.update(resolved) + return self._resolve_schema_references(schema, resolver) + + if "properties" in schema: + for k, val in schema["properties"].items(): + schema["properties"][k] = self._resolve_schema_references(val, resolver) + + if "patternProperties" in schema: + for k, val in schema["patternProperties"].items(): + schema["patternProperties"][k] = self._resolve_schema_references(val, resolver) + + if "items" in schema: + schema["items"] = self._resolve_schema_references(schema["items"], resolver) + + if "anyOf" in schema: + for i, element in enumerate(schema["anyOf"]): + schema["anyOf"][i] = self._resolve_schema_references(element, resolver) + + return schema + + def resolve(self, schema: dict, refs: Dict[str, dict] = None) -> dict: + """Resolves and replaces json-schema $refs with the appropriate dict. + Recursively walks the given schema dict, converting every instance + of $ref in a 'properties' structure with a resolved dict. + This modifies the input schema and also returns it. + Arguments: + schema: + the schema dict + refs: + a dict of which forms a store of referenced schemata + Returns: + schema + """ + refs = refs or {} + refs = {**self._shared_refs, **refs} + return self._resolve_schema_references(schema, RefResolver("", schema, store=refs)) + + class ResourceSchemaLoader: """JSONSchema loader from package resources""" @@ -46,6 +108,9 @@ def __init__(self, package_name: str): def get_schema(self, name: str) -> dict: raw_schema = json.loads(pkgutil.get_data(self.package_name, f"schemas/{name}.json")) + shared_schemas_folder = pkg_resources.resource_filename(self.package_name, "schemas/shared/") + if os.path.exists(shared_schemas_folder): + return JsonSchemaResolver(shared_schemas_folder).resolve(raw_schema) return raw_schema @@ -70,13 +135,19 @@ def _enumerate_methods(self) -> Dict[str, callable]: return mapping + @staticmethod + def _get_fields_from_stream(stream: AirbyteStream) -> List[str]: + return list(stream.json_schema.get("properties", {}).keys()) + def read_stream(self, stream: AirbyteStream) -> Generator[AirbyteRecordMessage, None, None]: """Yield records from stream""" method = self._stream_methods.get(stream.name) if not method: raise ValueError(f"Client does not know how to read stream `{stream.name}`") - for message in method(): + fields = self._get_fields_from_stream(stream) + + for message in method(fields=fields): now = int(datetime.now().timestamp()) * 1000 yield AirbyteRecordMessage(stream=stream.name, data=message, emitted_at=now) diff --git a/airbyte-integrations/bases/base-python/setup.py b/airbyte-integrations/bases/base-python/setup.py index aecdc63eb7c6..91923a41df31 100644 --- a/airbyte-integrations/bases/base-python/setup.py +++ b/airbyte-integrations/bases/base-python/setup.py @@ -32,7 +32,7 @@ url="https://github.com/airbytehq/airbyte", packages=setuptools.find_packages(), package_data={"": ["models/yaml/*.yaml"]}, - install_requires=["PyYAML==5.3.1", "pydantic==1.6.1", "airbyte-protocol"], + install_requires=["PyYAML==5.3.1", "pydantic==1.6.1", "airbyte-protocol", "jsonschema==2.6.0"], entry_points={ "console_scripts": ["base-python=base_python.entrypoint:main"], }, diff --git a/airbyte-integrations/bases/base-singer/setup.py b/airbyte-integrations/bases/base-singer/setup.py index cbee89245be5..26ce603450a0 100644 --- a/airbyte-integrations/bases/base-singer/setup.py +++ b/airbyte-integrations/bases/base-singer/setup.py @@ -30,5 +30,5 @@ author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "base-singer"], + install_requires=["airbyte-protocol"], ) diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/StandardSourceTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/StandardSourceTest.java index 286686054be5..cca49f96fc4d 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/StandardSourceTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/StandardSourceTest.java @@ -277,11 +277,10 @@ public void testIdenticalFullRefreshes() throws Exception { final List recordMessagesSecondRun = filterRecords(runRead(configuredCatalog)); // the worker validates the messages, so we just validate the message, so we do not need to validate // again (as long as we use the worker, which we will not want to do long term). - final String assertionMessage = "Expected two full refresh syncs to produce the same records"; - assertFalse(recordMessagesFirstRun.isEmpty(), assertionMessage); - assertFalse(recordMessagesSecondRun.isEmpty(), assertionMessage); + assertFalse(recordMessagesFirstRun.isEmpty(), "Expected first full refresh to produce records"); + assertFalse(recordMessagesSecondRun.isEmpty(), "Expected second full refresh to produce records"); - assertSameRecords(recordMessagesFirstRun, recordMessagesSecondRun, assertionMessage); + assertSameRecords(recordMessagesFirstRun, recordMessagesSecondRun, "Expected two full refresh syncs to produce the same records"); } /** diff --git a/airbyte-integrations/connectors/source-drift/source_drift/client/client.py b/airbyte-integrations/connectors/source-drift/source_drift/client/client.py index f0e79767eadc..82d7d7bf1c82 100644 --- a/airbyte-integrations/connectors/source-drift/source_drift/client/client.py +++ b/airbyte-integrations/connectors/source-drift/source_drift/client/client.py @@ -35,13 +35,13 @@ def __init__(self, access_token: str): super().__init__() self._client = APIClient(access_token) - def stream__accounts(self) -> Iterator[dict]: + def stream__accounts(self, **kwargs) -> Iterator[dict]: yield from self._client.accounts.list() - def stream__users(self) -> Iterator[dict]: + def stream__users(self, **kwargs) -> Iterator[dict]: yield from self._client.users.list() - def stream__conversations(self) -> Iterator[dict]: + def stream__conversations(self, **kwargs) -> Iterator[dict]: yield from self._client.conversations.list() def health_check(self) -> Tuple[bool, str]: diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile new file mode 100644 index 000000000000..fc13777a81d6 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -0,0 +1,16 @@ +FROM airbyte/integration-base-python:dev + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +ENV CODE_PATH="source_facebook_marketing" +ENV AIRBYTE_IMPL_MODULE="source_facebook_marketing" +ENV AIRBYTE_IMPL_PATH="SourceFacebookMarketing" + +WORKDIR /airbyte/integration_code +COPY $CODE_PATH ./$CODE_PATH +COPY setup.py ./ +RUN pip install . + +LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile.test b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile.test new file mode 100644 index 000000000000..46ee9983ff97 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile.test @@ -0,0 +1,22 @@ +FROM airbyte/base-python-test:dev + +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +ENV CODE_PATH="integration_tests" +ENV AIRBYTE_TEST_MODULE="integration_tests" +ENV AIRBYTE_TEST_PATH="SourceFacebookMarketingStandardTest" + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-facebook-marketing-standard-test + +WORKDIR /airbyte/integration_code +COPY source_facebook_marketing source_facebook_marketing +COPY $CODE_PATH $CODE_PATH +COPY sample_files/*.json $CODE_PATH/ +COPY secrets/* $CODE_PATH +COPY source_facebook_marketing/*.json $CODE_PATH +COPY setup.py ./ + +RUN pip install ".[tests]" + +WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-facebook-marketing/README.md b/airbyte-integrations/connectors/source-facebook-marketing/README.md new file mode 100644 index 000000000000..aca586255d03 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/README.md @@ -0,0 +1,73 @@ +# Facebook Marketing Source + +This is the repository for the Facebook Marketing source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/facebook-marketing). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Build & Activate Virtual Environment +First, build the module by running the following from the `airbyte` project root directory: +``` +./gradlew :airbyte-integrations:connectors:source-facebook-marketing:build +``` + +This will generate a virtualenv for this module in `source-facebook-marketing/.venv`. Make sure this venv is active in your +development environment of choice. To activate the venv from the terminal, run: +``` +cd airbyte-integrations/connectors/source-facebook-marketing # cd into the connector directory +source .venv/bin/activate +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/facebook-marketing) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_facebook_marketing/spec.json` file. +See `sample_files/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in RPass under the secret name `source-facebook-marketing-integration-test-config` +and place them into `secrets/config.json`. + + +### Locally running the connector +``` +python main_dev.py spec +python main_dev.py check --config secrets/config.json +python main_dev.py discover --config secrets/config.json +python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json +``` + +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +pytest unit_tests +``` + +### Locally running the connector docker image +``` +# in airbyte root directory +./gradlew :airbyte-integrations:connectors:source-facebook-marketing:airbyteDocker +docker run --rm airbyte/source-facebook-marketing:dev spec +docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-facebook-marketing/secrets:/secrets airbyte/source-facebook-marketing:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-facebook-marketing/secrets:/secrets airbyte/source-facebook-marketing:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-facebook-marketing/secrets:/secrets -v $(pwd)/airbyte-integrations/connectors/source-facebook-marketing/sample_files:/sample_files airbyte/source-facebook-marketing:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +``` + +### Integration Tests +1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-facebook-marketing:standardSourceTestPython` to run the standard integration test suite. +1. To run additional integration tests, place your integration tests in the `integration_tests` directory and run them with `pytest integration_tests`. + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. + +## Populating account with the data + +The following will create 120 accounts and conversations +```bash +export DRIFT_TOKEN= +cd airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/ +python -m client.fixture +``` diff --git a/airbyte-integrations/connectors/source-facebook-marketing/build.gradle b/airbyte-integrations/connectors/source-facebook-marketing/build.gradle new file mode 100644 index 000000000000..ed74b5faa2cd --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/build.gradle @@ -0,0 +1,13 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-test' +} + +airbytePython { + moduleDirectory 'source_facebook_marketing' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/__init__.py b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/__init__.py new file mode 100644 index 000000000000..a5d70f66da6f --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .standard_source_test import SourceFacebookMarketingStandardTest + +__all__ = ["SourceFacebookMarketingStandardTest"] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/standard_source_test.py new file mode 100644 index 000000000000..0236ae0544eb --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/standard_source_test.py @@ -0,0 +1,29 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from base_python_test import DefaultStandardSourceTest + + +class SourceFacebookMarketingStandardTest(DefaultStandardSourceTest): + pass diff --git a/airbyte-integrations/connectors/source-facebook-marketing/main_dev.py b/airbyte-integrations/connectors/source-facebook-marketing/main_dev.py new file mode 100644 index 000000000000..1390da4449d2 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/main_dev.py @@ -0,0 +1,32 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import sys + +from base_python.entrypoint import launch +from source_facebook_marketing import SourceFacebookMarketing + +if __name__ == "__main__": + source = SourceFacebookMarketing() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/requirements.txt b/airbyte-integrations/connectors/source-facebook-marketing/requirements.txt new file mode 100644 index 000000000000..76af767f3755 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/requirements.txt @@ -0,0 +1,4 @@ +-e ../../bases/airbyte-protocol +-e ../../bases/base-python +-e ../../bases/base-python-test +-e . diff --git a/airbyte-integrations/connectors/source-facebook-marketing/sample_files/catalog.json b/airbyte-integrations/connectors/source-facebook-marketing/sample_files/catalog.json new file mode 100644 index 000000000000..9917063c93be --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/sample_files/catalog.json @@ -0,0 +1,217 @@ +{ + "streams": [ + { + "name": "campaigns", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "name": { + "type": ["null", "string"] + }, + "objective": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "account_id": { + "type": ["null", "string"] + }, + "effective_status": { + "type": ["null", "string"] + }, + "buying_type": { + "type": ["null", "string"] + }, + "spend_cap": { + "type": ["null", "number"] + }, + "start_time": { + "type": "string", + "format": "date-time" + }, + "updated_time": { + "type": "string", + "format": "date-time" + }, + "ads": { + "properties": { + "data": { + "items": { + "properties": { + "id": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + } + }, + "type": ["null", "object"] + }, + "adlabels": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "updated_time": { + "type": "string", + "format": "date-time" + } + } + } + } + }, + "type": ["null", "object"] + }, + "supported_sync_modes": ["full_refresh"] + }, + { + "name": "adsets", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "promoted_object": { + "type": ["null", "object"], + "properties": { + "custom_event_type": { + "type": ["null", "string"] + }, + "pixel_id": { + "type": ["null", "string"] + }, + "pixel_rule": { + "type": ["null", "string"] + }, + "page_id": { + "type": ["null", "string"] + }, + "object_store_url": { + "type": ["null", "string"] + }, + "application_id": { + "type": ["null", "string"] + }, + "product_set_id": { + "type": ["null", "string"] + }, + "offer_id": { + "type": ["null", "string"] + } + } + }, + "id": { + "type": ["null", "string"] + }, + "account_id": { + "type": ["null", "string"] + }, + "updated_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "daily_budget": { + "type": ["null", "number"], + "maximum": 100000000000000000000000000000000, + "minimum": -100000000000000000000000000000000, + "multipleOf": 0.000001, + "exclusiveMaximum": true, + "exclusiveMinimum": true + }, + "budget_remaining": { + "type": ["null", "number"], + "maximum": 100000000000000000000000000000000, + "minimum": -100000000000000000000000000000000, + "multipleOf": 0.000001, + "exclusiveMaximum": true, + "exclusiveMinimum": true + }, + "effective_status": { + "type": ["null", "string"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "created_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "start_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "lifetime_budget": { + "type": ["null", "number"], + "maximum": 100000000000000000000000000000000, + "minimum": -100000000000000000000000000000000, + "multipleOf": 0.000001, + "exclusiveMaximum": true, + "exclusiveMinimum": true + }, + "targeting": { + "$ref": "targeting.json" + }, + "bid_info": { + "type": ["null", "object"], + "properties": { + "CLICKS": { + "type": ["null", "integer"] + }, + "ACTIONS": { + "type": ["null", "integer"] + }, + "IMPRESSIONS": { + "type": ["null", "integer"] + }, + "REACH": { + "type": ["null", "integer"] + } + } + }, + "adlabels": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "updated_time": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "supported_sync_modes": ["full_refresh"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-facebook-marketing/sample_files/configured_catalog.json new file mode 100644 index 000000000000..f7ee74eae091 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/sample_files/configured_catalog.json @@ -0,0 +1,221 @@ +{ + "streams": [ + { + "stream": { + "name": "campaigns", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "name": { + "type": ["null", "string"] + }, + "objective": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "account_id": { + "type": ["null", "string"] + }, + "effective_status": { + "type": ["null", "string"] + }, + "buying_type": { + "type": ["null", "string"] + }, + "spend_cap": { + "type": ["null", "number"] + }, + "start_time": { + "type": "string", + "format": "date-time" + }, + "updated_time": { + "type": "string", + "format": "date-time" + }, + "ads": { + "properties": { + "data": { + "items": { + "properties": { + "id": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + } + }, + "type": ["null", "object"] + }, + "adlabels": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "updated_time": { + "type": "string", + "format": "date-time" + } + } + } + } + }, + "type": ["null", "object"] + }, + "supported_sync_modes": ["full_refresh"] + } + }, + { + "stream": { + "name": "adsets", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "promoted_object": { + "type": ["null", "object"], + "properties": { + "custom_event_type": { + "type": ["null", "string"] + }, + "pixel_id": { + "type": ["null", "string"] + }, + "pixel_rule": { + "type": ["null", "string"] + }, + "page_id": { + "type": ["null", "string"] + }, + "object_store_url": { + "type": ["null", "string"] + }, + "application_id": { + "type": ["null", "string"] + }, + "product_set_id": { + "type": ["null", "string"] + }, + "offer_id": { + "type": ["null", "string"] + } + } + }, + "id": { + "type": ["null", "string"] + }, + "account_id": { + "type": ["null", "string"] + }, + "updated_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "daily_budget": { + "type": ["null", "number"], + "maximum": 100000000000000000000000000000000, + "minimum": -100000000000000000000000000000000, + "multipleOf": 0.000001, + "exclusiveMaximum": true, + "exclusiveMinimum": true + }, + "budget_remaining": { + "type": ["null", "number"], + "maximum": 100000000000000000000000000000000, + "minimum": -100000000000000000000000000000000, + "multipleOf": 0.000001, + "exclusiveMaximum": true, + "exclusiveMinimum": true + }, + "effective_status": { + "type": ["null", "string"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "created_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "start_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "lifetime_budget": { + "type": ["null", "number"], + "maximum": 100000000000000000000000000000000, + "minimum": -100000000000000000000000000000000, + "multipleOf": 0.000001, + "exclusiveMaximum": true, + "exclusiveMinimum": true + }, + "targeting": { + "$ref": "targeting.json" + }, + "bid_info": { + "type": ["null", "object"], + "properties": { + "CLICKS": { + "type": ["null", "integer"] + }, + "ACTIONS": { + "type": ["null", "integer"] + }, + "IMPRESSIONS": { + "type": ["null", "integer"] + }, + "REACH": { + "type": ["null", "integer"] + } + } + }, + "adlabels": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "updated_time": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "supported_sync_modes": ["full_refresh"] + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/sample_files/sample_config.json b/airbyte-integrations/connectors/source-facebook-marketing/sample_files/sample_config.json new file mode 100644 index 000000000000..1a27cebe9805 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/sample_files/sample_config.json @@ -0,0 +1,5 @@ +{ + "start_date": "2020-09-25T00:00:00Z", + "account_id": "", + "access_token": "" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/setup.py b/airbyte-integrations/connectors/source-facebook-marketing/setup.py new file mode 100644 index 000000000000..77abc7c34a61 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/setup.py @@ -0,0 +1,46 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from setuptools import find_packages, setup + +setup( + name="source_facebook_marketing", + description="Source implementation for Facebook Marketing.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=[ + "airbyte-protocol==0.0.0", + "base-python==0.0.0", + "facebook_business==9.0.1", + "backoff==1.10.0", + "python-dateutil==2.8.1", + "cached_property==1.5.2", + ], + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + tests_require=["pytest==6.1.2"], + extras_require={ + "tests": ["airbyte_python_test==0.0.0", "pytest==6.1.2"], + }, +) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/__init__.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/__init__.py new file mode 100644 index 000000000000..2cf6d9f0394c --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/__init__.py @@ -0,0 +1,28 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .client import Client +from .source import SourceFacebookMarketing + +__all__ = ["SourceFacebookMarketing", "Client"] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/client/__init__.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/client/__init__.py new file mode 100644 index 000000000000..981f9d11203d --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/client/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .client import Client, FacebookAPIException + +__all__ = ["Client", "FacebookAPIException"] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/client/client.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/client/client.py new file mode 100644 index 000000000000..f31e631eeeef --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/client/client.py @@ -0,0 +1,300 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from datetime import datetime +from typing import Iterator, Sequence, Tuple + +import backoff +from base_python import BaseClient +from base_python.entrypoint import logger # FIXME (Eugene K): register logger as standard python logger +from cached_property import cached_property +from dateutil.parser import isoparse +from facebook_business import FacebookAdsApi +from facebook_business.adobjects import user as fb_user +from facebook_business.exceptions import FacebookRequestError + +from .common import FacebookAPIException, retry_pattern + + +class StreamAPI: + result_return_limit = 100 + + def __init__(self, api): + self._api = api + + def list(self, fields: Sequence[str] = None) -> Iterator[dict]: + raise NotImplementedError + + +class AdCreativeAPI(StreamAPI): + """AdCreative is not an iterable stream as it uses the batch endpoint + doc: https://developers.facebook.com/docs/marketing-api/reference/adgroup/adcreatives/ + """ + + BATCH_SIZE = 50 + + def list(self, fields: Sequence[str] = None) -> Iterator[dict]: + ad_creative = self._get_creatives() + + # Create the initial batch + api_batch = self._api.new_batch() + records = [] + + def success(response): + records.append(response) + + def failure(response): + raise response.error() + + # This loop syncs minimal AdCreative objects + for i, creative in enumerate(ad_creative): + # Execute and create a new batch for every BATCH_SIZE added + if i % self.BATCH_SIZE == 0: + api_batch.execute() + api_batch = self._api.new_batch() + yield from records + records[:] = [] + + # Add a call to the batch with the full object + creative.api_get(fields=fields, batch=api_batch, success=success, failure=failure) + + # Ensure the final batch is executed + api_batch.execute() + + @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5) + def _get_creatives(self): + return self._api.account.get_ad_creatives(params={"limit": self.result_return_limit}) + + +class AdsAPI(StreamAPI): + """ + doc: https://developers.facebook.com/docs/marketing-api/reference/adgroup + """ + + def list(self, fields: Sequence[str] = None) -> Iterator[dict]: + ads = self._get_ads({"limit": self.result_return_limit}) + for recordset in ads: + for record in recordset: + yield self._extend_record(record, fields=fields) + + @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5) + def _get_ads(self, params): + """ + This is necessary because the functions that call this endpoint return + a generator, whose calls need decorated with a backoff. + """ + return self._api.account.get_ads(params=params) + + @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5) + def _extend_record(self, ad, fields): + return ad.api_get(fields=fields).export_all_data() + + +class AdSetsAPI(StreamAPI): + """ doc: https://developers.facebook.com/docs/marketing-api/reference/ad-campaign """ + + def list(self, fields: Sequence[str] = None) -> Iterator[dict]: + adsets = self._get_ad_sets({"limit": self.result_return_limit}) + + for adset in adsets: + yield self._extend_record(adset, fields=fields) + + @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5) + def _get_ad_sets(self, params): + """ + This is necessary because the functions that call this endpoint return + a generator, whose calls need decorated with a backoff. + """ + return self._api.account.get_ad_sets(params=params) + + @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5) + def _extend_record(self, ad_set, fields): + return ad_set.api_get(fields=fields).export_all_data() + + +class CampaignAPI(StreamAPI): + def list(self, fields: Sequence[str] = None) -> Iterator[dict]: + """Read available campaigns""" + pull_ads = "ads" in fields + fields = [k for k in fields if k != "ads"] + campaigns = self._get_campaigns({"limit": self.result_return_limit}) + for campaign in campaigns: + yield self._extend_record(campaign, fields=fields, pull_ads=pull_ads) + + @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5) + def _extend_record(self, campaign, fields, pull_ads): + """Request additional attributes for campaign""" + campaign_out = campaign.api_get(fields=fields).export_all_data() + if pull_ads: + campaign_out["ads"] = {"data": []} + ids = [ad["id"] for ad in campaign.get_ads()] + for ad_id in ids: + campaign_out["ads"]["data"].append({"id": ad_id}) + return campaign_out + + @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5) + def _get_campaigns(self, params): + """Separate method to request list of campaigns + This is necessary because the functions that call this endpoint return + a generator, whose calls need decorated with a backoff. + """ + return self._api.account.get_campaigns(params=params) + + +class AdsInsightAPI(StreamAPI): + ALL_ACTION_ATTRIBUTION_WINDOWS = [ + "1d_click", + "7d_click", + "28d_click", + "1d_view", + "7d_view", + "28d_view", + ] + + ALL_ACTION_BREAKDOWNS = [ + "action_type", + "action_target_id", + "action_destination", + ] + + # Some automatic fields (primary-keys) cannot be used as 'fields' query params. + INVALID_INSIGHT_FIELDS = [ + "impression_device", + "publisher_platform", + "platform_position", + "age", + "gender", + "country", + "placement", + "region", + "dma", + ] + + action_breakdowns = ALL_ACTION_BREAKDOWNS + level = "ad" + action_attribution_windows = ALL_ACTION_ATTRIBUTION_WINDOWS + time_increment = 1 + buffer_days = 28 + + def __init__(self, api, start_date, breakdowns=None): + super().__init__(api=api) + self.start_date = start_date + self.breakdowns = breakdowns + + def list(self, fields: Sequence[str] = None) -> Iterator[dict]: + for params in self._params(): + for obj in self._get_insights(params): + rec = obj.export_all_data() + yield rec + + def _params(self, fields: Sequence[str] = None) -> Iterator[dict]: + buffered_start_date = self.start_date.subtract(days=self.buffer_days) + end_date = datetime.now() + + fields = list(set(fields) - set(self.INVALID_INSIGHT_FIELDS)) + + while buffered_start_date <= end_date: + yield { + "level": self.level, + "action_breakdowns": self.action_breakdowns, + "breakdowns": self.breakdowns, + "limit": self.result_return_limit, + "fields": fields, + "time_increment": self.time_increment, + "action_attribution_windows": self.action_attribution_windows, + "time_ranges": [{"since": buffered_start_date.to_date_string(), "until": buffered_start_date.to_date_string()}], + } + buffered_start_date = buffered_start_date.add(days=1) + + @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5) + def _get_insights(self, params): + return self._api.account.get_insights(params=params) + + +class Client(BaseClient): + def __init__(self, account_id: str, access_token: str, start_date: str, include_deleted: bool = False): + super().__init__() + self._api = FacebookAdsApi.init(access_token=access_token) + self._account_id = account_id + self._start_date = isoparse(start_date) + self._include_deleted = include_deleted + + @cached_property + def account(self): + return self._find_account(self._account_id) + + def stream__campaigns(self, fields=None, **kwargs): + yield from CampaignAPI(self).list(fields) + + def stream__adsets(self, fields=None, **kwargs): + yield from AdSetsAPI(self).list(fields) + + def stream__ads(self, fields=None, **kwargs): + yield from AdsAPI(self).list(fields) + + def stream__adcreatives(self, fields=None, **kwargs): + yield from AdCreativeAPI(self).list(fields) + + def stream__ads_insights(self, fields=None, **kwargs): + client = AdsInsightAPI(self, start_date=self._start_date, **kwargs) + yield from client.list(fields) + + def stream__ads_insights_age_and_gender(self, fields=None, **kwargs): + yield from self.stream__ads_insights(fields=fields, breakdowns=["age", "gender"]) + + def stream__ads_insights_country(self, fields=None, **kwargs): + yield from self.stream__ads_insights(fields=fields, breakdowns=["country"]) + + def stream__ads_insights_platform_and_device(self, fields=None, **kwargs): + yield from self.stream__ads_insights(fields=fields, breakdowns=["publisher_platform", "platform_position", "impression_device"]) + + def stream__ads_insights_region(self, fields=None, **kwargs): + yield from self.stream__ads_insights(fields=fields, breakdowns=["region"]) + + def stream__ads_insights_dma(self, fields=None, **kwargs): + yield from self.stream__ads_insights(fields=fields, breakdowns=["dma"]) + + @staticmethod + def _find_account(account_id: str): + try: + accounts = fb_user.User(fbid="me").get_ad_accounts() + for account in accounts: + if account["account_id"] == account_id: + return account + except FacebookRequestError as exc: + raise FacebookAPIException(f"Error: {exc.api_error_code()}, {exc.api_error_message()}") from exc + + raise FacebookAPIException("Couldn't find account with id {}".format(account_id)) + + def health_check(self) -> Tuple[bool, str]: + alive = True + error_message = None + try: + self._find_account(self._account_id) + except FacebookAPIException as exc: + logger.error(str(exc)) # we might need some extra details, so log original exception here + alive = False + error_message = str(exc) + + return alive, error_message diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/client/common.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/client/common.py new file mode 100644 index 000000000000..97dca735f55d --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/client/common.py @@ -0,0 +1,56 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import sys + +import backoff +from base_python.entrypoint import logger # FIXME (Eugene K): register logger as standard python logger +from facebook_business.exceptions import FacebookRequestError + +FACEBOOK_UNKNOWN_ERROR_CODE = 99 + + +class FacebookAPIException(Exception): + """General class for all API errors""" + + +def retry_pattern(backoff_type, exception, **wait_gen_kwargs): + def log_retry_attempt(details): + _, exc, _ = sys.exc_info() + logger.info(str(exc)) + logger.info(f"Caught retryable error after {details['tries']} tries. Waiting {details['wait']} more seconds then retrying...") + + def should_retry_api_error(exc): + if isinstance(exc, FacebookRequestError): + return exc.api_transient_error() or exc.api_error_subcode() == FACEBOOK_UNKNOWN_ERROR_CODE + return False + + return backoff.on_exception( + backoff_type, + exception, + jitter=None, + on_backoff=log_retry_attempt, + giveup=lambda exc: not should_retry_api_error(exc), + **wait_gen_kwargs, + ) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/adcreatives.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/adcreatives.json new file mode 100644 index 000000000000..eb443dab21a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/adcreatives.json @@ -0,0 +1,855 @@ +{ + "properties": { + "body": { + "type": ["null", "string"] + }, + "object_story_id": { + "type": ["null", "string"] + }, + "image_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "account_id": { + "type": ["null", "string"] + }, + "actor_id": { + "type": ["null", "string"] + }, + "adlabels": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "name": { + "type": "string" + }, + "updated_time": { + "type": "string", + "format": "date-time" + } + } + } + }, + "applink_treatment": { + "type": ["null", "string"] + }, + "call_to_action_type": { + "type": ["null", "string"] + }, + "effective_instagram_story_id": { + "type": ["null", "string"] + }, + "effective_object_story_id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "image_crops": { "$ref": "ads_image_crops.json" }, + "instagram_actor_id": { + "type": ["null", "string"] + }, + "instagram_permalink_url": { + "type": ["null", "string"] + }, + "instagram_story_id": { + "type": ["null", "string"] + }, + "link_og_id": { + "type": ["null", "string"] + }, + "object_id": { + "type": ["null", "string"] + }, + "object_story_spec": { + "properties": { + "page_id": { + "type": ["null", "string"] + }, + "instagram_actor_id": { + "type": ["null", "string"] + }, + "link_data": { + "properties": { + "additional_image_index": { + "type": ["null", "integer"] + }, + "app_link_spec": { + "type": ["null", "object"], + "properties": { + "android": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "app_name": { + "type": "string" + }, + "class": { + "type": "string" + }, + "package": { + "type": "string" + }, + "url": { + "type": "string" + } + } + } + }, + "ios": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "app_name": { + "type": "string" + }, + "app_store_id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + } + }, + "ipad": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "app_name": { + "type": "string" + }, + "app_store_id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + } + }, + "iphone": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "app_name": { + "type": "string" + }, + "app_store_id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + } + } + } + }, + "attachment_style": { + "type": ["null", "string"] + }, + "branded_content_sponsor_page_id": { + "type": ["null", "string"] + }, + "branded_content_sponsor_relationship": { + "type": ["null", "string"] + }, + "call_to_action": { + "properties": { + "value": { + "properties": { + "app_destination": { + "type": ["null", "string"] + }, + "app_link": { + "type": ["null", "string"] + }, + "application": { + "type": ["null", "string"] + }, + "event_id": { + "type": ["null", "string"] + }, + "lead_gen_form_id": { + "type": ["null", "string"] + }, + "link": { + "type": ["null", "string"] + }, + "link_caption": { + "type": ["null", "string"] + }, + "link_format": { + "type": ["null", "string"] + }, + "page": { + "type": ["null", "string"] + }, + "product_link": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "caption": { + "type": ["null", "string"] + }, + "child_attachments": { + "items": { + "properties": { + "image_hash": { + "type": ["null", "string"] + }, + "link": { + "type": ["null", "string"] + }, + "call_to_action": { + "properties": { + "value": { + "properties": { + "app_destination": { + "type": ["null", "string"] + }, + "app_link": { + "type": ["null", "string"] + }, + "application": { + "type": ["null", "string"] + }, + "event_id": { + "type": ["null", "string"] + }, + "lead_gen_form_id": { + "type": ["null", "string"] + }, + "link": { + "type": ["null", "string"] + }, + "link_caption": { + "type": ["null", "string"] + }, + "link_format": { + "type": ["null", "string"] + }, + "page": { + "type": ["null", "string"] + }, + "product_link": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "caption": { + "type": ["null", "string"] + }, + "picture": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "image_crops": { "$ref": "ads_image_crops.json" }, + "name": { + "type": ["null", "string"] + }, + "static_card": { + "type": ["null", "boolean"] + }, + "video_id": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "multi_share_optimized": { + "type": ["null", "boolean"] + }, + "link": { + "type": ["null", "string"] + }, + "image_crops": { "$ref": "ads_image_crops.json" }, + "description": { + "type": ["null", "string"] + }, + "event_id": { + "type": ["null", "string"] + }, + "force_single_link": { + "type": ["null", "boolean"] + }, + "multi_share_end_card": { + "type": ["null", "boolean"] + }, + "message": { + "type": ["null", "string"] + }, + "image_hash": { + "type": ["null", "string"] + }, + "picture": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "offer_id": { + "type": ["null", "string"] + }, + "page_welcome_message": { + "type": ["null", "string"] + }, + "retailer_item_ids": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "show_multiple_images": { + "type": ["null", "boolean"] + } + }, + "type": ["null", "object"] + }, + "photo_data": { + "type": ["null", "object"], + "properties": { + "branded_content_sponsor_page_id": { + "type": ["null", "string"] + }, + "branded_content_sponsor_relationship": { + "type": ["null", "string"] + }, + "caption": { + "type": "string" + }, + "image_hash": { + "type": ["null", "string"] + }, + "page_welcome_message": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "template_data": { + "properties": { + "additional_image_index": { + "type": ["null", "integer"] + }, + "app_link_spec": { + "type": ["null", "object"], + "properties": { + "android": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "app_name": { + "type": "string" + }, + "class": { + "type": "string" + }, + "package": { + "type": "string" + }, + "url": { + "type": "string" + } + } + } + }, + "ios": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "app_name": { + "type": "string" + }, + "app_store_id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + } + }, + "ipad": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "app_name": { + "type": "string" + }, + "app_store_id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + } + }, + "iphone": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "app_name": { + "type": "string" + }, + "app_store_id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + } + } + } + }, + "attachment_style": { + "type": ["null", "string"] + }, + "branded_content_sponsor_page_id": { + "type": ["null", "string"] + }, + "branded_content_sponsor_relationship": { + "type": ["null", "string"] + }, + "call_to_action": { + "properties": { + "value": { + "properties": { + "app_destination": { + "type": ["null", "string"] + }, + "app_link": { + "type": ["null", "string"] + }, + "application": { + "type": ["null", "string"] + }, + "event_id": { + "type": ["null", "string"] + }, + "lead_gen_form_id": { + "type": ["null", "string"] + }, + "link": { + "type": ["null", "string"] + }, + "link_caption": { + "type": ["null", "string"] + }, + "link_format": { + "type": ["null", "string"] + }, + "page": { + "type": ["null", "string"] + }, + "product_link": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "caption": { + "type": ["null", "string"] + }, + "child_attachments": { + "items": { + "properties": { + "image_hash": { + "type": ["null", "string"] + }, + "link": { + "type": ["null", "string"] + }, + "call_to_action": { + "properties": { + "value": { + "properties": { + "app_destination": { + "type": ["null", "string"] + }, + "app_link": { + "type": ["null", "string"] + }, + "application": { + "type": ["null", "string"] + }, + "event_id": { + "type": ["null", "string"] + }, + "lead_gen_form_id": { + "type": ["null", "string"] + }, + "link": { + "type": ["null", "string"] + }, + "link_caption": { + "type": ["null", "string"] + }, + "link_format": { + "type": ["null", "string"] + }, + "page": { + "type": ["null", "string"] + }, + "product_link": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "caption": { + "type": ["null", "string"] + }, + "picture": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "image_crops": { "$ref": "ads_image_crops.json" }, + "name": { + "type": ["null", "string"] + }, + "static_card": { + "type": ["null", "boolean"] + }, + "video_id": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "multi_share_optimized": { + "type": ["null", "boolean"] + }, + "link": { + "type": ["null", "string"] + }, + "image_crops": { "$ref": "ads_image_crops.json" }, + "description": { + "type": ["null", "string"] + }, + "event_id": { + "type": ["null", "string"] + }, + "force_single_link": { + "type": ["null", "boolean"] + }, + "multi_share_end_card": { + "type": ["null", "boolean"] + }, + "message": { + "type": ["null", "string"] + }, + "image_hash": { + "type": ["null", "string"] + }, + "picture": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "offer_id": { + "type": ["null", "string"] + }, + "page_welcome_message": { + "type": ["null", "string"] + }, + "retailer_item_ids": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "show_multiple_images": { + "type": ["null", "boolean"] + } + }, + "type": ["null", "object"] + }, + "text_data": { + "type": ["null", "object"], + "properties": { + "message": { + "type": "string" + } + } + }, + "video_data": { + "type": ["null", "object"], + "properties": { + "additional_image_index": { + "type": ["null", "integer"] + }, + "branded_content_sponsor_page_id": { + "type": ["null", "string"] + }, + "branded_content_sponsor_relationship": { + "type": ["null", "string"] + }, + "call_to_action": { + "properties": { + "value": { + "properties": { + "app_destination": { + "type": ["null", "string"] + }, + "app_link": { + "type": ["null", "string"] + }, + "application": { + "type": ["null", "string"] + }, + "event_id": { + "type": ["null", "string"] + }, + "lead_gen_form_id": { + "type": ["null", "string"] + }, + "link": { + "type": ["null", "string"] + }, + "link_caption": { + "type": ["null", "string"] + }, + "link_format": { + "type": ["null", "string"] + }, + "page": { + "type": ["null", "string"] + }, + "product_link": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "image_hash": { + "type": ["null", "string"] + }, + "image_url": { + "type": ["null", "string"] + }, + "link_description": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "offer_id": { + "type": ["null", "string"] + }, + "page_welcome_message": { + "type": ["null", "string"] + }, + "retailer_item_ids": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "targeting": { "$ref": "targeting.json" }, + "title": { + "type": ["null", "string"] + }, + "video_id": { + "type": ["null", "string"] + } + } + } + }, + "type": ["null", "object"] + }, + "object_type": { + "type": ["null", "string"] + }, + "object_url": { + "type": ["null", "string"] + }, + "product_set_id": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "template_url": { + "type": ["null", "string"] + }, + "template_url_spec": { + "type": ["null", "object"], + "properties": { + "android": { + "type": ["null", "object"], + "properties": { + "app_name": { + "type": "string" + }, + "package": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "config": { + "type": ["null", "object"], + "properties": { + "app_id": { + "type": "string" + } + } + }, + "ios": { + "type": ["null", "object"], + "properties": { + "app_name": { + "type": "string" + }, + "app_store_id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "ipad": { + "type": ["null", "object"], + "properties": { + "app_name": { + "type": "string" + }, + "app_store_id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "iphone": { + "type": ["null", "object"], + "properties": { + "app_name": { + "type": "string" + }, + "app_store_id": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "web": { + "type": ["null", "object"], + "properties": { + "should_fallback": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "windows_phone": { + "type": ["null", "object"], + "properties": { + "app_id": { + "type": "string" + }, + "app_name": { + "type": "string" + }, + "url": { + "type": "string" + } + } + } + } + }, + "thumbnail_url": { + "type": ["null", "string"] + }, + "image_hash": { + "type": ["null", "string"] + }, + "url_tags": { + "type": ["null", "string"] + }, + "video_id": { + "type": ["null", "string"] + }, + "link_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads.json new file mode 100644 index 000000000000..ce9dcb853fc8 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads.json @@ -0,0 +1,454 @@ +{ + "type": ["null", "object"], + "properties": { + "bid_type": { + "type": ["null", "string"] + }, + "account_id": { + "type": ["null", "string"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "adset_id": { + "type": ["null", "string"] + }, + "adlabels": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "name": { + "type": "string" + }, + "updated_time": { + "type": "string", + "format": "date-time" + } + } + } + }, + "bid_amount": { + "type": ["null", "integer"] + }, + "bid_info": { + "type": ["null", "object"], + "properties": { + "CLICKS": { + "type": ["null", "integer"] + }, + "ACTIONS": { + "type": ["null", "integer"] + }, + "REACH": { + "type": ["null", "integer"] + }, + "IMPRESSIONS": { + "type": ["null", "integer"] + }, + "SOCIAL": { + "type": ["null", "integer"] + } + } + }, + "status": { + "type": ["null", "string"] + }, + "creative": { + "type": ["null", "object"], + "properties": { + "creative_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + } + } + }, + "id": { + "type": ["null", "string"] + }, + "updated_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "name": { + "type": ["null", "string"] + }, + "targeting": { "$ref": "targeting.json" }, + "effective_status": { + "type": ["null", "string"] + }, + "last_updated_by_app_id": { + "type": ["null", "string"] + }, + "recommendations": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "blame_field": { + "type": ["null", "string"] + }, + "code": { + "type": "integer" + }, + "confidence": { + "type": "string" + }, + "importance": { + "type": "string" + }, + "message": { + "type": "string" + }, + "title": { + "type": "string" + } + } + } + }, + "source_ad_id": { + "type": ["null", "string"] + }, + "tracking_specs": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "application": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "post": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "conversion_id": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "action.type": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "post.wall": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "page": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "creative": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "dataset": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "event": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "event.creator": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "event_type": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "fb_pixel": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "fb_pixel_event": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "leadgen": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "object": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "object.domain": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "offer": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "offer.creator": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "offsite_pixel": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "page.parent": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "post.object": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "post.object.wall": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "question": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "question.creator": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "response": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "subtype": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + } + }, + "conversion_specs": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "application": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "post": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "conversion_id": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "action.type": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "post.wall": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "page": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "creative": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "dataset": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "event": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "event.creator": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "event_type": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "fb_pixel": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "fb_pixel_event": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "leadgen": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "object": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "object.domain": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "offer": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "offer.creator": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "offsite_pixel": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "page.parent": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "post.object": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "post.object.wall": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "question": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "question.creator": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "response": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "subtype": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json new file mode 100644 index 000000000000..9e5ea7186e90 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json @@ -0,0 +1,179 @@ +{ + "type": ["null", "object"], + "properties": { + "unique_actions": { "$ref": "ads_action_stats.json" }, + "actions": { "$ref": "ads_action_stats.json" }, + "action_values": { "$ref": "ads_action_stats.json" }, + "outbound_clicks": { "$ref": "ads_action_stats.json" }, + "video_30_sec_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p25_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p50_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p75_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p100_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_play_curve_actions": { "$ref": "ads_histogram_stats.json" }, + "clicks": { + "type": ["null", "integer"] + }, + "date_stop": { + "type": ["null", "string"], + "format": "date-time" + }, + "ad_id": { + "type": ["null", "string"] + }, + "website_ctr": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "number"] + }, + "action_destination": { + "type": ["null", "string"] + }, + "action_target_id": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "unique_inline_link_click_ctr": { + "type": ["null", "number"] + }, + "adset_id": { + "type": ["null", "string"] + }, + "frequency": { + "type": ["null", "number"] + }, + "account_name": { + "type": ["null", "string"] + }, + "canvas_avg_view_time": { + "type": ["null", "number"] + }, + "unique_inline_link_clicks": { + "type": ["null", "integer"] + }, + "cost_per_unique_action_type": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "inline_post_engagement": { + "type": ["null", "integer"] + }, + "campaign_name": { + "type": ["null", "string"] + }, + "inline_link_clicks": { + "type": ["null", "integer"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "cpc": { + "type": ["null", "number"] + }, + "ad_name": { + "type": ["null", "string"] + }, + "cost_per_unique_inline_link_click": { + "type": ["null", "number"] + }, + "cpm": { + "type": ["null", "number"] + }, + "cost_per_inline_post_engagement": { + "type": ["null", "number"] + }, + "inline_link_click_ctr": { + "type": ["null", "number"] + }, + "cpp": { + "type": ["null", "number"] + }, + "cost_per_action_type": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "unique_link_clicks_ctr": { + "type": ["null", "number"] + }, + "spend": { + "type": ["null", "number"] + }, + "cost_per_unique_click": { + "type": ["null", "number"] + }, + "adset_name": { + "type": ["null", "string"] + }, + "unique_clicks": { + "type": ["null", "integer"] + }, + "social_spend": { + "type": ["null", "number"] + }, + "reach": { + "type": ["null", "integer"] + }, + "canvas_avg_view_percent": { + "type": ["null", "number"] + }, + "account_id": { + "type": ["null", "string"] + }, + "date_start": { + "type": ["null", "string"], + "format": "date-time" + }, + "objective": { + "type": ["null", "string"] + }, + "quality_ranking": { + "type": ["null", "string"] + }, + "engagement_rate_ranking": { + "type": ["null", "string"] + }, + "conversion_rate_ranking": { + "type": ["null", "string"] + }, + "impressions": { + "type": ["null", "integer"] + }, + "unique_ctr": { + "type": ["null", "number"] + }, + "cost_per_inline_link_click": { + "type": ["null", "number"] + }, + "ctr": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_age_and_gender.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_age_and_gender.json new file mode 100644 index 000000000000..0de28798b28b --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_age_and_gender.json @@ -0,0 +1,186 @@ +{ + "type": ["null", "object"], + "properties": { + "unique_actions": { "$ref": "ads_action_stats.json" }, + "actions": { "$ref": "ads_action_stats.json" }, + "action_values": { "$ref": "ads_action_stats.json" }, + "outbound_clicks": { "$ref": "ads_action_stats.json" }, + "video_30_sec_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p25_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p50_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p75_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p100_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_play_curve_actions": { "$ref": "ads_histogram_stats.json" }, + "clicks": { + "type": ["null", "integer"] + }, + "date_stop": { + "type": ["null", "string"] + }, + "ad_id": { + "type": ["null", "string"] + }, + "website_ctr": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "number"] + }, + "action_destination": { + "type": ["null", "string"] + }, + "action_target_id": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "unique_inline_link_click_ctr": { + "type": ["null", "number"] + }, + "adset_id": { + "type": ["null", "string"] + }, + "frequency": { + "type": ["null", "number"] + }, + "account_name": { + "type": ["null", "string"] + }, + "canvas_avg_view_time": { + "type": ["null", "number"] + }, + "unique_inline_link_clicks": { + "type": ["null", "integer"] + }, + "cost_per_unique_action_type": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "inline_post_engagement": { + "type": ["null", "integer"] + }, + "campaign_name": { + "type": ["null", "string"] + }, + "inline_link_clicks": { + "type": ["null", "integer"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "cpc": { + "type": ["null", "number"] + }, + "ad_name": { + "type": ["null", "string"] + }, + "cost_per_unique_inline_link_click": { + "type": ["null", "number"] + }, + "cpm": { + "type": ["null", "number"] + }, + "cost_per_inline_post_engagement": { + "type": ["null", "number"] + }, + "inline_link_click_ctr": { + "type": ["null", "number"] + }, + "cpp": { + "type": ["null", "number"] + }, + "cost_per_action_type": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "unique_link_clicks_ctr": { + "type": ["null", "number"] + }, + "spend": { + "type": ["null", "number"] + }, + "cost_per_unique_click": { + "type": ["null", "number"] + }, + "adset_name": { + "type": ["null", "string"] + }, + "unique_clicks": { + "type": ["null", "integer"] + }, + "social_spend": { + "type": ["null", "number"] + }, + "reach": { + "type": ["null", "integer"] + }, + "canvas_avg_view_percent": { + "type": ["null", "number"] + }, + "account_id": { + "type": ["null", "string"] + }, + "date_start": { + "type": ["null", "string"] + }, + "objective": { + "type": ["null", "string"] + }, + "impressions": { + "type": ["null", "integer"] + }, + "unique_ctr": { + "type": ["null", "number"] + }, + "cost_per_inline_link_click": { + "type": ["null", "number"] + }, + "ctr": { + "type": ["null", "number"] + }, + "reach": { + "type": ["null", "integer"] + }, + "age": { + "type": ["null", "integer", "string"] + }, + "gender": { + "type": ["null", "string"] + }, + "quality_ranking": { + "type": ["null", "string"] + }, + "engagement_rate_ranking": { + "type": ["null", "string"] + }, + "conversion_rate_ranking": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_country.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_country.json new file mode 100644 index 000000000000..40614ab2a32b --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_country.json @@ -0,0 +1,185 @@ +{ + "type": ["null", "object"], + "properties": { + "unique_actions": { "$ref": "ads_action_stats.json" }, + "actions": { "$ref": "ads_action_stats.json" }, + "action_values": { "$ref": "ads_action_stats.json" }, + "outbound_clicks": { "$ref": "ads_action_stats.json" }, + "video_30_sec_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p25_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p50_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p75_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p100_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_play_curve_actions": { "$ref": "ads_histogram_stats.json" }, + "clicks": { + "type": ["null", "integer"] + }, + "date_stop": { + "type": ["null", "string"], + "format": "date-time" + }, + "ad_id": { + "type": ["null", "string"] + }, + "website_ctr": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "number"] + }, + "action_destination": { + "type": ["null", "string"] + }, + "action_target_id": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "unique_inline_link_click_ctr": { + "type": ["null", "number"] + }, + "adset_id": { + "type": ["null", "string"] + }, + "frequency": { + "type": ["null", "number"] + }, + "account_name": { + "type": ["null", "string"] + }, + "canvas_avg_view_time": { + "type": ["null", "number"] + }, + "unique_inline_link_clicks": { + "type": ["null", "integer"] + }, + "cost_per_unique_action_type": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "inline_post_engagement": { + "type": ["null", "integer"] + }, + "campaign_name": { + "type": ["null", "string"] + }, + "inline_link_clicks": { + "type": ["null", "integer"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "cpc": { + "type": ["null", "number"] + }, + "ad_name": { + "type": ["null", "string"] + }, + "cost_per_unique_inline_link_click": { + "type": ["null", "number"] + }, + "cpm": { + "type": ["null", "number"] + }, + "cost_per_inline_post_engagement": { + "type": ["null", "number"] + }, + "inline_link_click_ctr": { + "type": ["null", "number"] + }, + "cpp": { + "type": ["null", "number"] + }, + "cost_per_action_type": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "unique_link_clicks_ctr": { + "type": ["null", "number"] + }, + "spend": { + "type": ["null", "number"] + }, + "cost_per_unique_click": { + "type": ["null", "number"] + }, + "adset_name": { + "type": ["null", "string"] + }, + "unique_clicks": { + "type": ["null", "integer"] + }, + "social_spend": { + "type": ["null", "number"] + }, + "reach": { + "type": ["null", "integer"] + }, + "canvas_avg_view_percent": { + "type": ["null", "number"] + }, + "account_id": { + "type": ["null", "string"] + }, + "date_start": { + "type": ["null", "string"], + "format": "date-time" + }, + "objective": { + "type": ["null", "string"] + }, + "impressions": { + "type": ["null", "integer"] + }, + "unique_ctr": { + "type": ["null", "number"] + }, + "cost_per_inline_link_click": { + "type": ["null", "number"] + }, + "ctr": { + "type": ["null", "number"] + }, + "reach": { + "type": ["null", "integer"] + }, + "country": { + "type": ["null", "string"] + }, + "quality_ranking": { + "type": ["null", "string"] + }, + "engagement_rate_ranking": { + "type": ["null", "string"] + }, + "conversion_rate_ranking": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_dma.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_dma.json new file mode 100644 index 000000000000..f6a59632004f --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_dma.json @@ -0,0 +1,185 @@ +{ + "type": ["null", "object"], + "properties": { + "unique_actions": { "$ref": "ads_action_stats.json" }, + "actions": { "$ref": "ads_action_stats.json" }, + "action_values": { "$ref": "ads_action_stats.json" }, + "outbound_clicks": { "$ref": "ads_action_stats.json" }, + "video_30_sec_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p25_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p50_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p75_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p100_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_play_curve_actions": { "$ref": "ads_histogram_stats.json" }, + "clicks": { + "type": ["null", "integer"] + }, + "date_stop": { + "type": ["null", "string"], + "format": "date-time" + }, + "ad_id": { + "type": ["null", "string"] + }, + "website_ctr": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "number"] + }, + "action_destination": { + "type": ["null", "string"] + }, + "action_target_id": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "unique_inline_link_click_ctr": { + "type": ["null", "number"] + }, + "adset_id": { + "type": ["null", "string"] + }, + "frequency": { + "type": ["null", "number"] + }, + "account_name": { + "type": ["null", "string"] + }, + "canvas_avg_view_time": { + "type": ["null", "number"] + }, + "unique_inline_link_clicks": { + "type": ["null", "integer"] + }, + "cost_per_unique_action_type": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "inline_post_engagement": { + "type": ["null", "integer"] + }, + "campaign_name": { + "type": ["null", "string"] + }, + "inline_link_clicks": { + "type": ["null", "integer"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "cpc": { + "type": ["null", "number"] + }, + "ad_name": { + "type": ["null", "string"] + }, + "cost_per_unique_inline_link_click": { + "type": ["null", "number"] + }, + "cpm": { + "type": ["null", "number"] + }, + "cost_per_inline_post_engagement": { + "type": ["null", "number"] + }, + "inline_link_click_ctr": { + "type": ["null", "number"] + }, + "cpp": { + "type": ["null", "number"] + }, + "cost_per_action_type": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "unique_link_clicks_ctr": { + "type": ["null", "number"] + }, + "spend": { + "type": ["null", "number"] + }, + "cost_per_unique_click": { + "type": ["null", "number"] + }, + "adset_name": { + "type": ["null", "string"] + }, + "unique_clicks": { + "type": ["null", "integer"] + }, + "social_spend": { + "type": ["null", "number"] + }, + "reach": { + "type": ["null", "integer"] + }, + "canvas_avg_view_percent": { + "type": ["null", "number"] + }, + "account_id": { + "type": ["null", "string"] + }, + "date_start": { + "type": ["null", "string"], + "format": "date-time" + }, + "objective": { + "type": ["null", "string"] + }, + "impressions": { + "type": ["null", "integer"] + }, + "unique_ctr": { + "type": ["null", "number"] + }, + "cost_per_inline_link_click": { + "type": ["null", "number"] + }, + "ctr": { + "type": ["null", "number"] + }, + "reach": { + "type": ["null", "integer"] + }, + "dma": { + "type": ["null", "string"] + }, + "quality_ranking": { + "type": ["null", "string"] + }, + "engagement_rate_ranking": { + "type": ["null", "string"] + }, + "conversion_rate_ranking": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_platform_and_device.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_platform_and_device.json new file mode 100644 index 000000000000..ca2bfe85708e --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_platform_and_device.json @@ -0,0 +1,194 @@ +{ + "type": ["null", "object"], + "properties": { + "unique_actions": { "$ref": "ads_action_stats.json" }, + "actions": { "$ref": "ads_action_stats.json" }, + "action_values": { "$ref": "ads_action_stats.json" }, + "outbound_clicks": { "$ref": "ads_action_stats.json" }, + "video_30_sec_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p25_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p50_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p75_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p100_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_play_curve_actions": { "$ref": "ads_histogram_stats.json" }, + "impression_device": { + "type": ["null", "string"] + }, + "platform_position": { + "type": ["null", "string"] + }, + "publisher_platform": { + "type": ["null", "string"] + }, + "clicks": { + "type": ["null", "integer"] + }, + "date_stop": { + "type": ["null", "string"], + "format": "date-time" + }, + "ad_id": { + "type": ["null", "string"] + }, + "website_ctr": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "number"] + }, + "action_destination": { + "type": ["null", "string"] + }, + "action_target_id": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "unique_inline_link_click_ctr": { + "type": ["null", "number"] + }, + "adset_id": { + "type": ["null", "string"] + }, + "frequency": { + "type": ["null", "number"] + }, + "account_name": { + "type": ["null", "string"] + }, + "canvas_avg_view_time": { + "type": ["null", "number"] + }, + "unique_inline_link_clicks": { + "type": ["null", "integer"] + }, + "cost_per_unique_action_type": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "inline_post_engagement": { + "type": ["null", "integer"] + }, + "campaign_name": { + "type": ["null", "string"] + }, + "inline_link_clicks": { + "type": ["null", "integer"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "cpc": { + "type": ["null", "number"] + }, + "ad_name": { + "type": ["null", "string"] + }, + "cost_per_unique_inline_link_click": { + "type": ["null", "number"] + }, + "cpm": { + "type": ["null", "number"] + }, + "cost_per_inline_post_engagement": { + "type": ["null", "number"] + }, + "inline_link_click_ctr": { + "type": ["null", "number"] + }, + "cpp": { + "type": ["null", "number"] + }, + "cost_per_action_type": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "unique_link_clicks_ctr": { + "type": ["null", "number"] + }, + "spend": { + "type": ["null", "number"] + }, + "cost_per_unique_click": { + "type": ["null", "number"] + }, + "adset_name": { + "type": ["null", "string"] + }, + "unique_clicks": { + "type": ["null", "integer"] + }, + "social_spend": { + "type": ["null", "number"] + }, + "reach": { + "type": ["null", "integer"] + }, + "canvas_avg_view_percent": { + "type": ["null", "number"] + }, + "account_id": { + "type": ["null", "string"] + }, + "date_start": { + "type": ["null", "string"], + "format": "date-time" + }, + "objective": { + "type": ["null", "string"] + }, + "impressions": { + "type": ["null", "integer"] + }, + "unique_ctr": { + "type": ["null", "number"] + }, + "cost_per_inline_link_click": { + "type": ["null", "number"] + }, + "ctr": { + "type": ["null", "number"] + }, + "reach": { + "type": ["null", "integer"] + }, + "placement": { + "type": ["null", "string"] + }, + "quality_ranking": { + "type": ["null", "string"] + }, + "engagement_rate_ranking": { + "type": ["null", "string"] + }, + "conversion_rate_ranking": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_region.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_region.json new file mode 100644 index 000000000000..ec841bde2847 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_region.json @@ -0,0 +1,185 @@ +{ + "type": ["null", "object"], + "properties": { + "unique_actions": { "$ref": "ads_action_stats.json" }, + "actions": { "$ref": "ads_action_stats.json" }, + "action_values": { "$ref": "ads_action_stats.json" }, + "outbound_clicks": { "$ref": "ads_action_stats.json" }, + "video_30_sec_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p25_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p50_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p75_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_p100_watched_actions": { "$ref": "ads_action_stats.json" }, + "video_play_curve_actions": { "$ref": "ads_histogram_stats.json" }, + "clicks": { + "type": ["null", "integer"] + }, + "date_stop": { + "type": ["null", "string"], + "format": "date-time" + }, + "ad_id": { + "type": ["null", "string"] + }, + "website_ctr": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "number"] + }, + "action_destination": { + "type": ["null", "string"] + }, + "action_target_id": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "unique_inline_link_click_ctr": { + "type": ["null", "number"] + }, + "adset_id": { + "type": ["null", "string"] + }, + "frequency": { + "type": ["null", "number"] + }, + "account_name": { + "type": ["null", "string"] + }, + "canvas_avg_view_time": { + "type": ["null", "number"] + }, + "unique_inline_link_clicks": { + "type": ["null", "integer"] + }, + "cost_per_unique_action_type": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "inline_post_engagement": { + "type": ["null", "integer"] + }, + "campaign_name": { + "type": ["null", "string"] + }, + "inline_link_clicks": { + "type": ["null", "integer"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "cpc": { + "type": ["null", "number"] + }, + "ad_name": { + "type": ["null", "string"] + }, + "cost_per_unique_inline_link_click": { + "type": ["null", "number"] + }, + "cpm": { + "type": ["null", "number"] + }, + "cost_per_inline_post_engagement": { + "type": ["null", "number"] + }, + "inline_link_click_ctr": { + "type": ["null", "number"] + }, + "cpp": { + "type": ["null", "number"] + }, + "cost_per_action_type": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + } + } + } + }, + "unique_link_clicks_ctr": { + "type": ["null", "number"] + }, + "spend": { + "type": ["null", "number"] + }, + "cost_per_unique_click": { + "type": ["null", "number"] + }, + "adset_name": { + "type": ["null", "string"] + }, + "unique_clicks": { + "type": ["null", "integer"] + }, + "social_spend": { + "type": ["null", "number"] + }, + "reach": { + "type": ["null", "integer"] + }, + "canvas_avg_view_percent": { + "type": ["null", "number"] + }, + "account_id": { + "type": ["null", "string"] + }, + "date_start": { + "type": ["null", "string"], + "format": "date-time" + }, + "objective": { + "type": ["null", "string"] + }, + "impressions": { + "type": ["null", "integer"] + }, + "unique_ctr": { + "type": ["null", "number"] + }, + "cost_per_inline_link_click": { + "type": ["null", "number"] + }, + "ctr": { + "type": ["null", "number"] + }, + "reach": { + "type": ["null", "integer"] + }, + "region": { + "type": ["null", "string"] + }, + "quality_ranking": { + "type": ["null", "string"] + }, + "engagement_rate_ranking": { + "type": ["null", "string"] + }, + "conversion_rate_ranking": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/adsets.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/adsets.json new file mode 100644 index 000000000000..b5ad2fcae6f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/adsets.json @@ -0,0 +1,119 @@ +{ + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "promoted_object": { + "type": ["null", "object"], + "properties": { + "custom_event_type": { + "type": ["null", "string"] + }, + "pixel_id": { + "type": ["null", "string"] + }, + "pixel_rule": { + "type": ["null", "string"] + }, + "page_id": { + "type": ["null", "string"] + }, + "object_store_url": { + "type": ["null", "string"] + }, + "application_id": { + "type": ["null", "string"] + }, + "product_set_id": { + "type": ["null", "string"] + }, + "offer_id": { + "type": ["null", "string"] + } + } + }, + "id": { + "type": ["null", "string"] + }, + "account_id": { + "type": ["null", "string"] + }, + "updated_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "daily_budget": { + "type": ["null", "number"], + "maximum": 100000000000000000000000000000000, + "minimum": -100000000000000000000000000000000, + "multipleOf": 0.000001, + "exclusiveMaximum": true, + "exclusiveMinimum": true + }, + "budget_remaining": { + "type": ["null", "number"], + "maximum": 100000000000000000000000000000000, + "minimum": -100000000000000000000000000000000, + "multipleOf": 0.000001, + "exclusiveMaximum": true, + "exclusiveMinimum": true + }, + "effective_status": { + "type": ["null", "string"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "created_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "start_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "lifetime_budget": { + "type": ["null", "number"], + "maximum": 100000000000000000000000000000000, + "minimum": -100000000000000000000000000000000, + "multipleOf": 0.000001, + "exclusiveMaximum": true, + "exclusiveMinimum": true + }, + "targeting": { "$ref": "targeting.json" }, + "bid_info": { + "type": ["null", "object"], + "properties": { + "CLICKS": { + "type": ["null", "integer"] + }, + "ACTIONS": { + "type": ["null", "integer"] + }, + "IMPRESSIONS": { + "type": ["null", "integer"] + }, + "REACH": { + "type": ["null", "integer"] + } + } + }, + "adlabels": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { "type": "string" }, + "name": { "type": "string" }, + "created_time": { "type": "string", "format": "date-time" }, + "updated_time": { "type": "string", "format": "date-time" } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json new file mode 100644 index 000000000000..5c3cc0c7b7d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json @@ -0,0 +1,72 @@ +{ + "properties": { + "name": { + "type": ["null", "string"] + }, + "objective": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "account_id": { + "type": ["null", "string"] + }, + "effective_status": { + "type": ["null", "string"] + }, + "buying_type": { + "type": ["null", "string"] + }, + "spend_cap": { + "type": ["null", "number"] + }, + "start_time": { + "type": "string", + "format": "date-time" + }, + "updated_time": { + "type": "string", + "format": "date-time" + }, + "ads": { + "properties": { + "data": { + "items": { + "properties": { + "id": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + } + }, + "type": ["null", "object"] + }, + "adlabels": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "updated_time": { + "type": "string", + "format": "date-time" + } + } + } + } + }, + "type": ["null", "object"] +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/ads_action_stats.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/ads_action_stats.json new file mode 100644 index 000000000000..4c0a0d139837 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/ads_action_stats.json @@ -0,0 +1,38 @@ +{ + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "1d_click": { + "type": ["null", "number"] + }, + "7d_click": { + "type": ["null", "number"] + }, + "28d_click": { + "type": ["null", "number"] + }, + "1d_view": { + "type": ["null", "number"] + }, + "7d_view": { + "type": ["null", "number"] + }, + "28d_view": { + "type": ["null", "number"] + }, + "action_destination": { + "type": ["null", "string"] + }, + "action_target_id": { + "type": ["null", "string"] + }, + "action_type": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "number"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/ads_histogram_stats.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/ads_histogram_stats.json new file mode 100644 index 000000000000..1e1a0195a6b2 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/ads_histogram_stats.json @@ -0,0 +1,17 @@ +{ + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "action_type": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/ads_image_crops.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/ads_image_crops.json new file mode 100644 index 000000000000..051cc5e81cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/ads_image_crops.json @@ -0,0 +1,68 @@ +{ + "properties": { + "100x100": { + "items": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "type": ["null", "array"] + }, + "100x72": { + "items": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "type": ["null", "array"] + }, + "191x100": { + "items": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "type": ["null", "array"] + }, + "400x150": { + "items": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "type": ["null", "array"] + }, + "400x500": { + "items": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "type": ["null", "array"] + }, + "600x360": { + "items": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "type": ["null", "array"] + }, + "90x160": { + "items": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "type": ["null", "array"] + } + }, + "type": ["null", "object"] +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/geo_locations.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/geo_locations.json new file mode 100644 index 000000000000..0ed0a397c38a --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/geo_locations.json @@ -0,0 +1,141 @@ +{ + "type": ["null", "object"], + "properties": { + "regions": { + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + } + } + }, + "type": ["null", "array"] + }, + "countries": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "cities": { + "items": { + "type": ["null", "object"], + "properties": { + "key": { + "type": ["null", "string"] + }, + "distance_unit": { + "type": ["null", "string"] + }, + "region": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "region_id": { + "type": ["null", "string"] + }, + "radius": { + "type": ["null", "integer"] + } + } + }, + "type": ["null", "array"] + }, + "location_types": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "zips": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "primary_city_id": { + "type": ["null", "integer"] + }, + "region_id": { + "type": ["null", "integer"] + } + } + } + }, + "custom_locations": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "address_string": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "distance_unit": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "longitude": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "primary_city_id": { + "type": ["null", "integer"] + }, + "radius": { + "type": ["null", "integer"] + }, + "region_id": { + "type": ["null", "integer"] + } + } + } + }, + "country_groups": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "geo-markets": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "key": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/targeting.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/targeting.json new file mode 100644 index 000000000000..62137a190919 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/shared/targeting.json @@ -0,0 +1,272 @@ +{ + "definitions": { + "id_name_pairs": { + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + } + } + }, + "type": ["null", "array"] + }, + "targeting_fields": { + "type": ["null", "object"], + "properties": { + "household_composition": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "home_type": { + "$ref$": "targeting.json#/definitions/id_name_pairs" + }, + "friends_of_connections": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "family_statuses": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "work_positions": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "education_majors": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "life_events": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "moms": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "custom_audiences": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "interests": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "behaviors": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "connections": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "excluded_connections": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "user_adclusters": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "income": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "excluded_custom_audiences": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "work_employers": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "industries": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "net_worth": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "relationship_statuses": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "generation": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "home_ownership": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "politics": { + "$ref": "targeting.json#/definitions/id_name_pairs" + } + } + } + }, + "type": ["null", "object"], + "properties": { + "messenger_positions": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "locales": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "connections": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "instagram_positions": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "audience_network_positions": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "exclusions": { + "$ref": "targeting.json#/definitions/targeting_fields" + }, + "interests": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "geo_locations": { + "$ref": "geo_locations.json" + }, + "excluded_geo_locations": { + "$ref": "geo_locations.json" + }, + "work_positions": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "education_statuses": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "publisher_platforms": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "age_max": { + "type": ["null", "integer"] + }, + "custom_audiences": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "device_platforms": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "age_min": { + "type": ["null", "integer"] + }, + "facebook_positions": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "excluded_connections": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "flexible_spec": { + "items": { + "$ref": "targeting.json#/definitions/targeting_fields" + }, + "type": ["null", "array"] + }, + "friends_of_connections": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "user_os": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "user_device": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "excluded_custom_audiences": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "excluded_publisher_categories": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "genders": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "targeting_optimization": { + "type": ["null", "string"] + }, + "user_adclusters": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "generation": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "behaviors": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "moms": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "home_ownership": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "office_type": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "home_type": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "family_statuses": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "relationship_statuses": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "industries": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "income": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "app_install_state": { + "type": ["null", "string"] + }, + "net_worth": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "politics": { + "$ref": "targeting.json#/definitions/id_name_pairs" + }, + "interested_in": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "excluded_user_device": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py new file mode 100644 index 000000000000..bc1b77bab368 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -0,0 +1,31 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from base_python import BaseSource + +from .client import Client + + +class SourceFacebookMarketing(BaseSource): + client_class = Client diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.json new file mode 100644 index 000000000000..4ff443d5662f --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.json @@ -0,0 +1,32 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/facebook-marketing", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Source Facebook Marketing", + "type": "object", + "required": ["start_date", "account_id", "access_token"], + "additionalProperties": false, + "properties": { + "start_date": { + "type": "string", + "description": "The date from which you'd like to replicate data for AdCreatives and AdInsights APIs, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", + "examples": ["2020-09-25T00:00:00Z"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + }, + "account_id": { + "type": "string", + "description": "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API." + }, + "access_token": { + "type": "string", + "description": "The value of the access token generated. See the docs for more information", + "airbyte_secret": true + }, + "include_deleted": { + "type": "boolean", + "description": "Include data from deleted campaigns, ads, and adsets.", + "default": "true" + } + } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py new file mode 100644 index 000000000000..6d04162936d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py @@ -0,0 +1,41 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import pytest +from airbyte_protocol import AirbyteStream +from source_facebook_marketing.client import Client, FacebookAPIException + + +def test__health_check_with_wrong_token(): + client = Client(account_id="wrong_account", access_token="wrong_key", start_date="2019-03-03T10:00") + alive, error = client.health_check() + + assert not alive + assert error == "Error: 190, Invalid OAuth access token." + + +def test__campaigns_with_wrong_token(): + client = Client(account_id="wrong_account", access_token="wrong_key", start_date="2019-03-03T10:00") + with pytest.raises(FacebookAPIException, match="Error: 190, Invalid OAuth access token"): + next(client.read_stream(AirbyteStream(name="campaigns", json_schema={}))) diff --git a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile b/airbyte-integrations/connectors/source-shopify-singer/Dockerfile index 2039250e8407..3210040d9c0c 100644 --- a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-shopify-singer/Dockerfile @@ -10,7 +10,6 @@ ENV AIRBYTE_IMPL_PATH="SourceShopifySinger" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install pip==20.3 RUN pip install tap-shopify==1.2.6 RUN pip install ".[main]" diff --git a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile.test b/airbyte-integrations/connectors/source-shopify-singer/Dockerfile.test index f7ba3a37e5fd..680fe6c61da2 100644 --- a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile.test +++ b/airbyte-integrations/connectors/source-shopify-singer/Dockerfile.test @@ -18,7 +18,6 @@ COPY secrets/config.json $CODE_PATH/config.json COPY $MODULE_NAME/*.json $CODE_PATH/ COPY setup.py ./ -RUN pip install pip==20.3 RUN pip install tap-shopify==1.2.6 RUN pip install ".[tests]" diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle index 030ca2c0837a..c17bc2cf9a43 100644 --- a/buildSrc/src/main/groovy/airbyte-python.gradle +++ b/buildSrc/src/main/groovy/airbyte-python.gradle @@ -23,7 +23,7 @@ class AirbytePythonPlugin implements Plugin { pip 'mypy:0.790' pip 'isort:5.6.4' pip 'pytest:6.1.2' - pip 'pip:20.3' + pip 'pip:20.2' } project.task('blackFormat', type: PythonTask) { diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 2b3beb475089..5181d73ffa45 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -17,6 +17,7 @@ * [Drift](integrations/sources/drift.md) * [Exchange Rates API](integrations/sources/exchangeratesapi-io.md) * [Facebook Marketing API](integrations/sources/facebook-marketing-api.md) + * [Facebook Marketing](integrations/sources/facebook-marketing.md) * [Files](integrations/sources/file.md) * [Freshdesk](integrations/sources/freshdesk.md) * [Intercom](integrations/sources/intercom.md) diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md new file mode 100644 index 000000000000..e65aa4995d19 --- /dev/null +++ b/docs/integrations/sources/facebook-marketing.md @@ -0,0 +1,85 @@ +# Facebook Marketing + +## Sync overview + +This source can sync data for the core Ad Campaign data available in the [Facebook Marketing API](https://developers.facebook.com/docs/marketing-api/campaign-structure): Campaigns, AdSets, Ads, and AdCreatives. +It can also sync [Ad Insights from the Reporting API](https://developers.facebook.com/docs/marketing-api/insights). + +### Output schema + +This Source is capable of syncing the following core Streams: + +* AdSets. [Facebook docs](https://developers.facebook.com/docs/marketing-api/reference/ad-campaign#fields) +* Ads. [Facebook docs](https://developers.facebook.com/docs/marketing-api/reference/adgroup#fields) +* AdCreatives. [Facebook docs](https://developers.facebook.com/docs/marketing-api/reference/ad-creative#fields) +* Campaigns. [Facebook docs](https://developers.facebook.com/docs/marketing-api/reference/ad-campaign-group#fields) + +The linked Facebook docs go into detail about the fields present on those streams. + +In addition, this source is capable of syncing ad insights as a stream. Ad insights can also be segmented by the following categories, where each segment is synced as a separate Airbyte stream: + +* Country +* DMA \(Designated Market Area\) +* Gender & Age +* Platform & Device +* Region + +The segmented streams contain entries of campaign/adset/ad combinations for each day broken down by the chosen segment. + +For more information, see the [Facebook Insights API documentation. ](https://developers.facebook.com/docs/marketing-api/reference/adgroup/insights/) + +### Data type mapping + +| Integration Type | Airbyte Type | Notes | +| :--- | :--- | :--- | +| `string` | `string` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | yes | | +| Incremental Sync | no | | + +### Performance considerations + +**Important note:** In order for data synced from your Facebook account to be up to date, you might need to apply with Facebook to upgrade your access token to the Ads Management Standard Tier as specified in the [Facebook Access documentation](https://developers.facebook.com/docs/marketing-api/access). Otherwise, Facebook might throttle Airbyte syncs, since the default tier \(Dev Access\) is heavily throttled by Facebook. + +Note that Airbyte can adapt to throttling from Facebook. In the worst case scenario syncs from Facebook will take longer to complete and data will be less fresh. + +## Getting started + +### Requirements + +* A Facebook Ad Account ID +* A Facebook App which has the Marketing API enabled +* A Facebook Marketing API Access Token + +### Setup guide + +### Facebook Ad Account ID + +Follow the [Facebook documentation for obtaining your Ad Account ID](https://www.facebook.com/business/help/1492627900875762) and keep that on hand. We'll need this ID to configure Facebook as a source in Airbyte. + +### Facebook App + +#### If you don't have a Facebook App + +Visit the [Facebook Developers App hub](https://developers.facebook.com/apps/) and create an App and choose "Manage Business Integrations" as the purpose of the app. Fill out the remaining fields to create your app, then follow along the "Enable the Marketing API for your app" section. + +#### Enable the Marketing API for your app + +From the App's Dashboard screen \(seen in the screenshot below\) enable the Marketing API for your app if it is not already setup. + +![](../../.gitbook/assets/screen-shot-2020-11-03-at-9.25.21-pm%20%284%29.png) + +### API Access Token + +In the App Dashboard screen, click Marketing API --> Tools on the left sidebar. Then highlight all the available token permissions \(`ads_management`, `ads_read`, `read_insights`\) and click "Get token". A long string of characters should appear in front of you; **this is the access token.** Copy this string for use in the Airbyte UI later. + +![](../../.gitbook/assets/screen-shot-2020-11-03-at-9.35.40-pm%20%284%29.png) + +With the Ad Account ID and API access token, you should be ready to start pulling data from the Facebook Marketing API. Head to the Airbyte UI to setup your source connector! diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 383582f566d7..71e4164a6c74 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -26,6 +26,7 @@ write_standard_creds source-braintree-singer "$BRAINTREE_TEST_CREDS" write_standard_creds source-drift "$DRIFT_INTEGRATION_TEST_CREDS" write_standard_creds source-file "$AWS_S3_INTEGRATION_TEST_CREDS" "aws.json" write_standard_creds source-freshdesk "$FRESHDESK_TEST_CREDS" +write_standard_creds source-facebook-marketing "$FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS" write_standard_creds source-facebook-marketing-api-singer "$FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS" # pull sample config. add in the access key. write to secrets. GH_CREDS=$(jq --arg v "$GH_INTEGRATION_TEST_CREDS" '.access_token = $v' airbyte-integrations/connectors/source-github-singer/config.sample.json)