Source facebook marketing (no-singer)

.github/workflows/test-command.yml

@@ -50,6 +50,7 @@ jobs:
           BIGQUERY_INTEGRATION_TEST_CREDS: ${{ secrets.BIGQUERY_INTEGRATION_TEST_CREDS }}
           BRAINTREE_TEST_CREDS: ${{ secrets.BRAINTREE_TEST_CREDS }}
           DRIFT_INTEGRATION_TEST_CREDS: ${{ secrets.DRIFT_INTEGRATION_TEST_CREDS }}
+          FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS }}
           FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS }}
           FRESHDESK_TEST_CREDS: ${{ secrets.FRESHDESK_TEST_CREDS }}
           GH_INTEGRATION_TEST_CREDS: ${{ secrets.GH_INTEGRATION_TEST_CREDS }}

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json

@@ -0,0 +1,7 @@
+{
+  "sourceDefinitionId": "e7778cfc-e97c-4458-9ecb-b4f2bba8946c",
+  "name": "Facebook Marketing",
+  "dockerRepository": "airbyte/source-facebook-marketing",
+  "dockerImageTag": "0.1.1",
+  "documentationUrl": "https://hub.docker.com/r/airbyte/source-facebook-marketing"
+}

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -73,11 +73,11 @@
   dockerRepository: airbyte/source-googleanalytics-singer
   dockerImageTag: 0.1.5
   documentationUrl: https://hub.docker.com/r/airbyte/source-googleanalytics-singer
-- sourceDefinitionId: 74d47f79-8d01-44ac-9755-f5eb0d7caacb
-  name: Facebook Marketing APIs
-  dockerRepository: airbyte/source-facebook-marketing-api-singer
-  dockerImageTag: 0.1.5
-  documentationUrl: https://hub.docker.com/r/airbyte/source-facebook-marketing-api-singer
+- sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
+  name: Facebook Marketing
+  dockerRepository: airbyte/source-facebook-marketing
+  dockerImageTag: 0.1.1
+  documentationUrl: https://hub.docker.com/r/airbyte/source-facebook-marketing
 - sourceDefinitionId: 57eb1576-8f52-463d-beb6-2e107cdf571d
   name: Hubspot
   dockerRepository: airbyte/source-hubspot-singer

airbyte-integrations/bases/base-python/Dockerfile

@@ -4,6 +4,7 @@ COPY --from=airbyte/integration-base:dev /airbyte /airbyte
 WORKDIR /airbyte/base_python_code
 COPY base_python ./base_python
 COPY setup.py ./
+RUN pip install pip==20.2
 RUN pip install .
 
 ENV AIRBYTE_SPEC_CMD "base-python spec"

airbyte-integrations/bases/base-python/base_python/client.py

@@ -24,12 +24,15 @@
 
 import inspect
 import json
+import os
 import pkgutil
 from abc import ABC, abstractmethod
 from datetime import datetime
-from typing import Dict, Generator, Tuple
+from typing import Dict, Generator, List, Tuple
 
+import pkg_resources
 from airbyte_protocol import AirbyteRecordMessage, AirbyteStream
+from jsonschema import RefResolver
 
 
 def package_name_from_class(cls: object) -> str:
@@ -38,6 +41,65 @@ def package_name_from_class(cls: object) -> str:
     return module.__name__.split(".")[0]
 
 
+class JsonSchemaResolver:
+    """Helper class to expand $ref items in json schema"""
+
+    def __init__(self, shared_schemas_path: str):
+        self._shared_refs = self._load_shared_schema_refs(shared_schemas_path)
+
+    @staticmethod
+    def _load_shared_schema_refs(path: str):
+        shared_file_names = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
+
+        shared_schema_refs = {}
+        for shared_file in shared_file_names:
+            with open(os.path.join(path, shared_file)) as data_file:
+                shared_schema_refs[shared_file] = json.load(data_file)
+
+        return shared_schema_refs
+
+    def _resolve_schema_references(self, schema: dict, resolver: RefResolver) -> dict:
+        if "$ref" in schema:
+            reference_path = schema.pop("$ref", None)
+            resolved = resolver.resolve(reference_path)[1]
+            schema.update(resolved)
+            return self._resolve_schema_references(schema, resolver)
+
+        if "properties" in schema:
+            for k, val in schema["properties"].items():
+                schema["properties"][k] = self._resolve_schema_references(val, resolver)
+
+        if "patternProperties" in schema:
+            for k, val in schema["patternProperties"].items():
+                schema["patternProperties"][k] = self._resolve_schema_references(val, resolver)
+
+        if "items" in schema:
+            schema["items"] = self._resolve_schema_references(schema["items"], resolver)
+
+        if "anyOf" in schema:
+            for i, element in enumerate(schema["anyOf"]):
+                schema["anyOf"][i] = self._resolve_schema_references(element, resolver)
+
+        return schema
+
+    def resolve(self, schema: dict, refs: Dict[str, dict] = None) -> dict:
+        """Resolves and replaces json-schema $refs with the appropriate dict.
+        Recursively walks the given schema dict, converting every instance
+        of $ref in a 'properties' structure with a resolved dict.
+        This modifies the input schema and also returns it.
+        Arguments:
+            schema:
+                the schema dict
+            refs:
+                a dict of <string, dict> which forms a store of referenced schemata
+        Returns:
+            schema
+        """
+        refs = refs or {}
+        refs = {**self._shared_refs, **refs}
+        return self._resolve_schema_references(schema, RefResolver("", schema, store=refs))
+
+
 class ResourceSchemaLoader:
     """JSONSchema loader from package resources"""
 
@@ -46,6 +108,9 @@ def __init__(self, package_name: str):
 
     def get_schema(self, name: str) -> dict:
         raw_schema = json.loads(pkgutil.get_data(self.package_name, f"schemas/{name}.json"))
+        shared_schemas_folder = pkg_resources.resource_filename(self.package_name, "schemas/shared/")
+        if os.path.exists(shared_schemas_folder):
+            return JsonSchemaResolver(shared_schemas_folder).resolve(raw_schema)
         return raw_schema
 
 
@@ -70,13 +135,19 @@ def _enumerate_methods(self) -> Dict[str, callable]:
 
         return mapping
 
+    @staticmethod
+    def _get_fields_from_stream(stream: AirbyteStream) -> List[str]:
+        return list(stream.json_schema.get("properties", {}).keys())
+
     def read_stream(self, stream: AirbyteStream) -> Generator[AirbyteRecordMessage, None, None]:
         """Yield records from stream"""
         method = self._stream_methods.get(stream.name)
         if not method:
             raise ValueError(f"Client does not know how to read stream `{stream.name}`")
 
-        for message in method():
+        fields = self._get_fields_from_stream(stream)
+
+        for message in method(fields=fields):
             now = int(datetime.now().timestamp()) * 1000
             yield AirbyteRecordMessage(stream=stream.name, data=message, emitted_at=now)
 

airbyte-integrations/bases/base-python/setup.py

@@ -32,7 +32,7 @@
     url="https://github.com/airbytehq/airbyte",
     packages=setuptools.find_packages(),
     package_data={"": ["models/yaml/*.yaml"]},
-    install_requires=["PyYAML==5.3.1", "pydantic==1.6.1", "airbyte-protocol"],
+    install_requires=["PyYAML==5.3.1", "pydantic==1.6.1", "airbyte-protocol", "jsonschema==2.6.0"],
     entry_points={
         "console_scripts": ["base-python=base_python.entrypoint:main"],
     },

airbyte-integrations/bases/base-singer/setup.py

@@ -30,5 +30,5 @@
     author="Airbyte",
     author_email="[email protected]",
     packages=find_packages(),
-    install_requires=["airbyte-protocol", "base-singer"],
+    install_requires=["airbyte-protocol"],
 )

airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/StandardSourceTest.java

@@ -277,11 +277,10 @@ public void testIdenticalFullRefreshes() throws Exception {
     final List<AirbyteRecordMessage> recordMessagesSecondRun = filterRecords(runRead(configuredCatalog));
     // the worker validates the messages, so we just validate the message, so we do not need to validate
     // again (as long as we use the worker, which we will not want to do long term).
-    final String assertionMessage = "Expected two full refresh syncs to produce the same records";
-    assertFalse(recordMessagesFirstRun.isEmpty(), assertionMessage);
-    assertFalse(recordMessagesSecondRun.isEmpty(), assertionMessage);
+    assertFalse(recordMessagesFirstRun.isEmpty(), "Expected first full refresh to produce records");
+    assertFalse(recordMessagesSecondRun.isEmpty(), "Expected second full refresh to produce records");
 
-    assertSameRecords(recordMessagesFirstRun, recordMessagesSecondRun, assertionMessage);
+    assertSameRecords(recordMessagesFirstRun, recordMessagesSecondRun, "Expected two full refresh syncs to produce the same records");
   }
 
   /**

airbyte-integrations/connectors/source-drift/source_drift/client/client.py

@@ -35,13 +35,13 @@ def __init__(self, access_token: str):
         super().__init__()
         self._client = APIClient(access_token)
 
-    def stream__accounts(self) -> Iterator[dict]:
+    def stream__accounts(self, **kwargs) -> Iterator[dict]:
         yield from self._client.accounts.list()
 
-    def stream__users(self) -> Iterator[dict]:
+    def stream__users(self, **kwargs) -> Iterator[dict]:
         yield from self._client.users.list()
 
-    def stream__conversations(self) -> Iterator[dict]:
+    def stream__conversations(self, **kwargs) -> Iterator[dict]:
         yield from self._client.conversations.list()
 
     def health_check(self) -> Tuple[bool, str]:

airbyte-integrations/connectors/source-facebook-marketing/Dockerfile

@@ -0,0 +1,16 @@
+FROM airbyte/integration-base-python:dev
+
+# Bash is installed for more convenient debugging.
+RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
+
+ENV CODE_PATH="source_facebook_marketing"
+ENV AIRBYTE_IMPL_MODULE="source_facebook_marketing"
+ENV AIRBYTE_IMPL_PATH="SourceFacebookMarketing"
+
+WORKDIR /airbyte/integration_code
+COPY $CODE_PATH ./$CODE_PATH
+COPY setup.py ./
+RUN pip install .
+
+LABEL io.airbyte.version=0.1.1
+LABEL io.airbyte.name=airbyte/source-facebook-marketing

airbyte-integrations/connectors/source-facebook-marketing/Dockerfile.test

@@ -0,0 +1,22 @@
+FROM airbyte/base-python-test:dev
+
+RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
+
+ENV CODE_PATH="integration_tests"
+ENV AIRBYTE_TEST_MODULE="integration_tests"
+ENV AIRBYTE_TEST_PATH="SourceFacebookMarketingStandardTest"
+
+LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.name=airbyte/source-facebook-marketing-standard-test
+
+WORKDIR /airbyte/integration_code
+COPY source_facebook_marketing source_facebook_marketing
+COPY $CODE_PATH $CODE_PATH
+COPY sample_files/*.json $CODE_PATH/
+COPY secrets/* $CODE_PATH
+COPY source_facebook_marketing/*.json $CODE_PATH
+COPY setup.py ./
+
+RUN pip install ".[tests]"
+
+WORKDIR /airbyte

airbyte-integrations/connectors/source-facebook-marketing/README.md

@@ -0,0 +1,73 @@
+# Facebook Marketing Source 
+
+This is the repository for the Facebook Marketing source connector, written in Python. 
+For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/facebook-marketing).
+
+## Local development
+
+### Prerequisites
+**To iterate on this connector, make sure to complete this prerequisites section.**
+
+#### Build & Activate Virtual Environment
+First, build the module by running the following from the `airbyte` project root directory: 
+```
+./gradlew :airbyte-integrations:connectors:source-facebook-marketing:build
+```
+
+This will generate a virtualenv for this module in `source-facebook-marketing/.venv`. Make sure this venv is active in your
+development environment of choice. To activate the venv from the terminal, run:
+```
+cd airbyte-integrations/connectors/source-facebook-marketing # cd into the connector directory
+source .venv/bin/activate
+```
+If you are in an IDE, follow your IDE's instructions to activate the virtualenv.
+
+#### Create credentials
+**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/facebook-marketing)
+to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_facebook_marketing/spec.json` file.
+See `sample_files/sample_config.json` for a sample config file.
+
+**If you are an Airbyte core member**, copy the credentials in RPass under the secret name `source-facebook-marketing-integration-test-config`
+and place them into `secrets/config.json`.
+
+
+### Locally running the connector
+```
+python main_dev.py spec
+python main_dev.py check --config secrets/config.json
+python main_dev.py discover --config secrets/config.json
+python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json
+```
+
+### Unit Tests
+To run unit tests locally, from the connector directory run:
+```
+pytest unit_tests
+```
+
+### Locally running the connector docker image
+```
+# in airbyte root directory
+./gradlew :airbyte-integrations:connectors:source-facebook-marketing:airbyteDocker
+docker run --rm airbyte/source-facebook-marketing:dev spec
+docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-facebook-marketing/secrets:/secrets airbyte/source-facebook-marketing:dev check --config /secrets/config.json
+docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-facebook-marketing/secrets:/secrets airbyte/source-facebook-marketing:dev discover --config /secrets/config.json
+docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-facebook-marketing/secrets:/secrets -v $(pwd)/airbyte-integrations/connectors/source-facebook-marketing/sample_files:/sample_files airbyte/source-facebook-marketing:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json
+```
+
+### Integration Tests
+1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-facebook-marketing:standardSourceTestPython` to run the standard integration test suite.
+1. To run additional integration tests, place your integration tests in the `integration_tests` directory and run them with `pytest integration_tests`.
+   Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
+
+## Dependency Management
+All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
+
+## Populating account with the data
+
+The following will create 120 accounts and conversations
+```bash
+export DRIFT_TOKEN=<PUT_YOUR_TOKEN_HERE>
+cd airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/
+python -m client.fixture
+```

airbyte-integrations/connectors/source-facebook-marketing/build.gradle

@@ -0,0 +1,13 @@
+plugins {
+    id 'airbyte-python'
+    id 'airbyte-docker'
+    id 'airbyte-source-test'
+}
+
+airbytePython {
+    moduleDirectory 'source_facebook_marketing'
+}
+
+dependencies {
+    implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
+}

airbyte-integrations/connectors/source-facebook-marketing/integration_tests/__init__.py

@@ -0,0 +1,27 @@
+"""
+MIT License
+
+Copyright (c) 2020 Airbyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+from .standard_source_test import SourceFacebookMarketingStandardTest
+
+__all__ = ["SourceFacebookMarketingStandardTest"]

airbyte-integrations/connectors/source-facebook-marketing/integration_tests/standard_source_test.py

@@ -0,0 +1,29 @@
+"""
+MIT License
+
+Copyright (c) 2020 Airbyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+from base_python_test import DefaultStandardSourceTest
+
+
+class SourceFacebookMarketingStandardTest(DefaultStandardSourceTest):
+    pass