diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/137ece28-5434-455c-8f34-69dc3782f451.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/137ece28-5434-455c-8f34-69dc3782f451.json index 92b84c2b5aad..d6b3a6beab53 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/137ece28-5434-455c-8f34-69dc3782f451.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/137ece28-5434-455c-8f34-69dc3782f451.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "137ece28-5434-455c-8f34-69dc3782f451", "name": "LinkedIn Ads", "dockerRepository": "airbyte/source-linkedin-ads", - "dockerImageTag": "0.1.0", + "dockerImageTag": "0.1.1", "documentationUrl": "https://docs.airbyte.io/integrations/sources/linkedin-ads" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 4ccee03705d0..24254a14bf97 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -538,7 +538,7 @@ - sourceDefinitionId: 137ece28-5434-455c-8f34-69dc3782f451 name: LinkedIn Ads dockerRepository: airbyte/source-linkedin-ads - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/linkedin-ads sourceType: api - sourceDefinitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e diff --git a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile index bddce18a5155..31a00a200351 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile @@ -5,7 +5,9 @@ FROM base as builder WORKDIR /airbyte/integration_code # upgrade pip to the latest version -RUN apk --no-cache upgrade && pip install --upgrade pip +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base COPY setup.py ./ # install necessary packages to a temporary folder @@ -17,15 +19,19 @@ WORKDIR /airbyte/integration_code # copy all loaded and built libraries to a pure basic image COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash # copy payload code only COPY main.py ./ COPY source_linkedin_ads ./source_linkedin_ads -# set the default Timezone, for use with dependent libraries like: datetime, pendullum, etc. -ENV TZ "UTC" ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-linkedin-ads diff --git a/airbyte-integrations/connectors/source-linkedin-ads/setup.py b/airbyte-integrations/connectors/source-linkedin-ads/setup.py index 231a23688bf7..1ae2b388c3fa 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/setup.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk==0.1.22", + "airbyte-cdk", "pendulum", ] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/creatives.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/creatives.json index 62559878dd8d..4b150ad2a33d 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/creatives.json +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/creatives.json @@ -50,24 +50,7 @@ "type": ["null", "string"] }, "value": { - "anyOf": [ - { - "type": ["null", "string"] - }, - { - "type": ["null", "boolean"] - }, - { - "type": ["null", "number"] - }, - { - "type": ["null", "integer"] - }, - { - "type": ["null", "object"], - "additionalProperties": true - } - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py index 368a183ed8bf..7f4161c83235 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py @@ -2,6 +2,7 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +import json from typing import Any, Dict, Iterable, List, Mapping import pendulum as pdm @@ -205,42 +206,53 @@ def transform_targeting_criteria( } """ - targeting_criteria = record.get(dict_key) - # transform `include` - if "include" in targeting_criteria: - and_list = targeting_criteria.get("include").get("and") - for id, and_criteria in enumerate(and_list): - or_dict = and_criteria.get("or") - for key, value in or_dict.items(): - values = [] - if isinstance(value, list): + + def unnest_dict(nested_dict: Dict) -> Iterable[Dict]: + """ + Unnest the nested dict to simplify the normalization + + EXAMPLE OUTPUT: + [ + {"type": "some_key", "values": "some_values"}, + ..., + {"type": "some_other_key", "values": "some_other_values"} + ] + """ + + for key, value in nested_dict.items(): + values = [] + if isinstance(value, List): + if len(value) > 0: if isinstance(value[0], str): values = value - elif isinstance(value[0], dict): + elif isinstance(value[0], Dict): for v in value: values.append(v) - elif isinstance(key, dict): - values.append(key) - # Replace the 'or' with {type:value} - record["targetingCriteria"]["include"]["and"][id]["type"] = key - record["targetingCriteria"]["include"]["and"][id]["values"] = values - record["targetingCriteria"]["include"]["and"][id].pop("or") + elif isinstance(value, Dict): + values.append(value) + yield {"type": key, "values": values} + + # get the target dict from record + targeting_criteria = record.get(dict_key) + + # transform `include` + if "include" in targeting_criteria: + and_list = targeting_criteria.get("include").get("and") + updated_include = {"and": []} + for k in and_list: + or_dict = k.get("or") + for j in unnest_dict(or_dict): + updated_include["and"].append(j) + # Replace the original 'and' with updated_include + record["targetingCriteria"]["include"] = updated_include # transform `exclude` if present if "exclude" in targeting_criteria: or_dict = targeting_criteria.get("exclude").get("or") updated_exclude = {"or": []} - for key, value in or_dict.items(): - values = [] - if isinstance(value, list): - if isinstance(value[0], str): - values = value - elif isinstance(value[0], dict): - for v in value: - value.append(v) - elif isinstance(value, dict): - value.append(value) - updated_exclude["or"].append({"type": key, "values": values}) + for k in unnest_dict(or_dict): + updated_exclude["or"].append(k) + # Replace the original 'or' with updated_exclude record["targetingCriteria"]["exclude"] = updated_exclude return record @@ -282,8 +294,9 @@ def transform_variables( for key, params in variables.items(): record["variables"]["type"] = key record["variables"]["values"] = [] - for key, param in params.items(): - record["variables"]["values"].append({"key": key, "value": param}) + for key, value in params.items(): + # convert various datatypes of values into the string + record["variables"]["values"].append({"key": key, "value": json.dumps(value, ensure_ascii=True)}) # Clean the nested structure record["variables"].pop("data") return record diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py index 07512a62f807..66cce8e60ea8 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py @@ -23,6 +23,8 @@ }, {"or": {"urn:li:adTargetingFacet:locations": ["urn:li:geo:103644278"]}}, {"or": {"urn:li:adTargetingFacet:interfaceLocales": ["urn:li:locale:en_US"]}}, + {"or": {"empty_dict_with_empty_list": []}}, # dict is present, but list is empty + {"or": {}}, # empty dict ] }, "exclude": { @@ -35,6 +37,7 @@ "facet_test3", "facet_test4", ], + "empty_list": [] } }, }, @@ -52,6 +55,10 @@ "activity": "urn:li:activity:1234", "directSponsoredContent": 0, "share": "urn:li:share:1234", + "custom_num_var": 1234, + "custom_obj_var": {"key": 1234}, + "custom_arr_var": [1, 2, 3, 4], + "custom_null_var": None, } } }, @@ -84,6 +91,10 @@ "type": "urn:li:adTargetingFacet:interfaceLocales", "values": ["urn:li:locale:en_US"], }, + { + "type": "empty_dict_with_empty_list", + "values": [], + }, ] }, "exclude": { @@ -96,15 +107,23 @@ "type": "urn:li:adTargetingFacet:facet_Key2", "values": ["facet_test3", "facet_test4"], }, + { + "type": "empty_list", + "values": [], + }, ] }, }, "variables": { "type": "com.linkedin.ads.SponsoredUpdateCreativeVariables", "values": [ - {"key": "activity", "value": "urn:li:activity:1234"}, - {"key": "directSponsoredContent", "value": 0}, - {"key": "share", "value": "urn:li:share:1234"}, + {"key": "activity", "value": '"urn:li:activity:1234"'}, + {"key": "directSponsoredContent", "value": "0"}, + {"key": "share", "value": '"urn:li:share:1234"'}, + {"key": "custom_num_var", "value": "1234"}, + {"key": "custom_obj_var", "value": '{"key": 1234}'}, + {"key": "custom_arr_var", "value": "[1, 2, 3, 4]"}, + {"key": "custom_null_var", "value": "null"}, ], }, "created": "2021-08-21 21:27:55", diff --git a/docs/integrations/sources/linkedin-ads.md b/docs/integrations/sources/linkedin-ads.md index c3917658c586..902b2e991cd6 100644 --- a/docs/integrations/sources/linkedin-ads.md +++ b/docs/integrations/sources/linkedin-ads.md @@ -132,4 +132,5 @@ The complete set of prmissions is: | Version | Date | Pull Request | Subject | | :------ | :-------- | :-------- | :------ | +| 0.1.1 | 2021-10-02 | [6610](https://github.com/airbytehq/airbyte/pull/6610) | Fix for `Campaigns/targetingCriteria` transformation, coerced `Creatives/variables/values` to string by default | | 0.1.0 | 2021-09-05 | [5285](https://github.com/airbytehq/airbyte/pull/5285) | Initial release of Native LinkedIn Ads connector for Airbyte |