🐛 Source LinkedIn-Ads: Fix for Campaigns/targetingCriteria transfor…

…mation, coerced `Creatives/variables/values` to string (#6610) * fixed targettingCriteria transformation, changed values of creatives/variables to string by default, bumped the version
airbytehq · Oct 6, 2021 · 64d7cf1 · 64d7cf1
1 parent 6e53a57
commit 64d7cf1
Show file tree

Hide file tree

Showing 8 changed files with 79 additions and 57 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/137ece28-5434-455c-8f34-69dc3782f451.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/137ece28-5434-455c-8f34-69dc3782f451.json
@@ -2,6 +2,6 @@
   "sourceDefinitionId": "137ece28-5434-455c-8f34-69dc3782f451",
   "name": "LinkedIn Ads",
   "dockerRepository": "airbyte/source-linkedin-ads",
-  "dockerImageTag": "0.1.0",
+  "dockerImageTag": "0.1.1",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/linkedin-ads"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -538,7 +538,7 @@
 - sourceDefinitionId: 137ece28-5434-455c-8f34-69dc3782f451
   name: LinkedIn Ads
   dockerRepository: airbyte/source-linkedin-ads
-  dockerImageTag: 0.1.0
+  dockerImageTag: 0.1.1
   documentationUrl: https://docs.airbyte.io/integrations/sources/linkedin-ads
   sourceType: api
 - sourceDefinitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e

diff --git a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile
@@ -5,7 +5,9 @@ FROM base as builder
 WORKDIR /airbyte/integration_code
 
 # upgrade pip to the latest version
-RUN apk --no-cache upgrade && pip install --upgrade pip
+RUN apk --no-cache upgrade \
+    && pip install --upgrade pip \
+    && apk --no-cache add tzdata build-base
 
 COPY setup.py ./
 # install necessary packages to a temporary folder 
@@ -17,15 +19,19 @@ WORKDIR /airbyte/integration_code
 
 # copy all loaded and built libraries to a pure basic image
 COPY --from=builder /install /usr/local
+# add default timezone settings
+COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime
+RUN echo "Etc/UTC" > /etc/timezone
+
+# bash is installed for more convenient debugging.
+RUN apk --no-cache add bash
 
 # copy payload code only
 COPY main.py ./
 COPY source_linkedin_ads ./source_linkedin_ads
 
-# set the default Timezone, for use with dependent libraries like: datetime, pendullum, etc.
-ENV TZ "UTC"
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.version=0.1.1
 LABEL io.airbyte.name=airbyte/source-linkedin-ads
diff --git a/airbyte-integrations/connectors/source-linkedin-ads/setup.py b/airbyte-integrations/connectors/source-linkedin-ads/setup.py
@@ -6,7 +6,7 @@
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
-    "airbyte-cdk==0.1.22",
+    "airbyte-cdk",
     "pendulum",
 ]
 

diff --git a/...te-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/creatives.json b/...te-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/creatives.json
@@ -50,24 +50,7 @@
                 "type": ["null", "string"]
               },
               "value": {
-                "anyOf": [
-                  {
-                    "type": ["null", "string"]
-                  },
-                  {
-                    "type": ["null", "boolean"]
-                  },
-                  {
-                    "type": ["null", "number"]
-                  },
-                  {
-                    "type": ["null", "integer"]
-                  },
-                  {
-                    "type": ["null", "object"],
-                    "additionalProperties": true
-                  }
-                ]
+                  "type": ["null", "string"]
               }
             }
           }

diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py
@@ -2,6 +2,7 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
+import json
 from typing import Any, Dict, Iterable, List, Mapping
 
 import pendulum as pdm
@@ -205,42 +206,53 @@ def transform_targeting_criteria(
             }
 
     """
-    targeting_criteria = record.get(dict_key)
-    # transform `include`
-    if "include" in targeting_criteria:
-        and_list = targeting_criteria.get("include").get("and")
-        for id, and_criteria in enumerate(and_list):
-            or_dict = and_criteria.get("or")
-            for key, value in or_dict.items():
-                values = []
-                if isinstance(value, list):
+
+    def unnest_dict(nested_dict: Dict) -> Iterable[Dict]:
+        """
+        Unnest the nested dict to simplify the normalization
+
+        EXAMPLE OUTPUT:
+            [
+                {"type": "some_key", "values": "some_values"},
+                ...,
+                {"type": "some_other_key", "values": "some_other_values"}
+            ]
+        """
+
+        for key, value in nested_dict.items():
+            values = []
+            if isinstance(value, List):
+                if len(value) > 0:
                     if isinstance(value[0], str):
                         values = value
-                    elif isinstance(value[0], dict):
+                    elif isinstance(value[0], Dict):
                         for v in value:
                             values.append(v)
-                elif isinstance(key, dict):
-                    values.append(key)
-                # Replace the 'or' with {type:value}
-                record["targetingCriteria"]["include"]["and"][id]["type"] = key
-                record["targetingCriteria"]["include"]["and"][id]["values"] = values
-                record["targetingCriteria"]["include"]["and"][id].pop("or")
+            elif isinstance(value, Dict):
+                values.append(value)
+            yield {"type": key, "values": values}
+
+    # get the target dict from record
+    targeting_criteria = record.get(dict_key)
+
+    # transform `include`
+    if "include" in targeting_criteria:
+        and_list = targeting_criteria.get("include").get("and")
+        updated_include = {"and": []}
+        for k in and_list:
+            or_dict = k.get("or")
+            for j in unnest_dict(or_dict):
+                updated_include["and"].append(j)
+        # Replace the original 'and' with updated_include
+        record["targetingCriteria"]["include"] = updated_include
 
     # transform `exclude` if present
     if "exclude" in targeting_criteria:
         or_dict = targeting_criteria.get("exclude").get("or")
         updated_exclude = {"or": []}
-        for key, value in or_dict.items():
-            values = []
-            if isinstance(value, list):
-                if isinstance(value[0], str):
-                    values = value
-                elif isinstance(value[0], dict):
-                    for v in value:
-                        value.append(v)
-            elif isinstance(value, dict):
-                value.append(value)
-            updated_exclude["or"].append({"type": key, "values": values})
+        for k in unnest_dict(or_dict):
+            updated_exclude["or"].append(k)
+        # Replace the original 'or' with updated_exclude
         record["targetingCriteria"]["exclude"] = updated_exclude
 
     return record
@@ -282,8 +294,9 @@ def transform_variables(
     for key, params in variables.items():
         record["variables"]["type"] = key
         record["variables"]["values"] = []
-        for key, param in params.items():
-            record["variables"]["values"].append({"key": key, "value": param})
+        for key, value in params.items():
+            # convert various datatypes of values into the string
+            record["variables"]["values"].append({"key": key, "value": json.dumps(value, ensure_ascii=True)})
         # Clean the nested structure
         record["variables"].pop("data")
     return record

diff --git a/...s/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py b/...s/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py
@@ -23,6 +23,8 @@
                     },
                     {"or": {"urn:li:adTargetingFacet:locations": ["urn:li:geo:103644278"]}},
                     {"or": {"urn:li:adTargetingFacet:interfaceLocales": ["urn:li:locale:en_US"]}},
+                    {"or": {"empty_dict_with_empty_list": []}}, # dict is present, but list is empty
+                    {"or": {}}, # empty dict
                 ]
             },
             "exclude": {
@@ -35,6 +37,7 @@
                         "facet_test3",
                         "facet_test4",
                     ],
+                    "empty_list": []
                 }
             },
         },
@@ -52,6 +55,10 @@
                     "activity": "urn:li:activity:1234",
                     "directSponsoredContent": 0,
                     "share": "urn:li:share:1234",
+                    "custom_num_var": 1234,
+                    "custom_obj_var": {"key": 1234},
+                    "custom_arr_var": [1, 2, 3, 4],
+                    "custom_null_var": None,
                 }
             }
         },
@@ -84,6 +91,10 @@
                         "type": "urn:li:adTargetingFacet:interfaceLocales",
                         "values": ["urn:li:locale:en_US"],
                     },
+                    {
+                        "type": "empty_dict_with_empty_list",
+                        "values": [],
+                    },
                 ]
             },
             "exclude": {
@@ -96,15 +107,23 @@
                         "type": "urn:li:adTargetingFacet:facet_Key2",
                         "values": ["facet_test3", "facet_test4"],
                     },
+                    {
+                        "type": "empty_list",
+                        "values": [],
+                    },
                 ]
             },
         },
         "variables": {
             "type": "com.linkedin.ads.SponsoredUpdateCreativeVariables",
             "values": [
-                {"key": "activity", "value": "urn:li:activity:1234"},
-                {"key": "directSponsoredContent", "value": 0},
-                {"key": "share", "value": "urn:li:share:1234"},
+                {"key": "activity", "value": '"urn:li:activity:1234"'},
+                {"key": "directSponsoredContent", "value": "0"},
+                {"key": "share", "value": '"urn:li:share:1234"'},
+                {"key": "custom_num_var", "value": "1234"},
+                {"key": "custom_obj_var", "value": '{"key": 1234}'},
+                {"key": "custom_arr_var", "value": "[1, 2, 3, 4]"},
+                {"key": "custom_null_var", "value": "null"},
             ],
         },
         "created": "2021-08-21 21:27:55",

diff --git a/docs/integrations/sources/linkedin-ads.md b/docs/integrations/sources/linkedin-ads.md
@@ -132,4 +132,5 @@ The complete set of prmissions is:
 
 | Version | Date       | Pull Request | Subject |
 | :------ | :--------  | :--------    | :------ |
+| 0.1.1   | 2021-10-02 | [6610](https://github.com/airbytehq/airbyte/pull/6610) | Fix for  `Campaigns/targetingCriteria` transformation, coerced  `Creatives/variables/values` to string by default |
 | 0.1.0   | 2021-09-05 | [5285](https://github.com/airbytehq/airbyte/pull/5285) | Initial release of Native LinkedIn Ads connector for Airbyte |