Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

attempted fix for hierarchy stream primary key #28

Merged
merged 21 commits into from
Apr 7, 2022
Merged

attempted fix for hierarchy stream primary key #28

Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
4976e08
attempted fix for hierarchy stream primary key
Mar 18, 2022
3941c6e
Merge pull request #1 from Widen/fix-pk
jlloyd-widen Mar 18, 2022
a9c54ef
overhauld of the primary key designations and flatten of schemas to m…
Mar 18, 2022
663ec2d
restored the meltano.yml file
Mar 18, 2022
02d9ffc
Merge pull request #2 from Widen/fix-pk
jlloyd-widen Mar 18, 2022
fc0e627
Revert "overhauld of the primary key designations and flatten of sche…
jlloyd-widen Mar 24, 2022
83dae8c
Merge pull request #3 from Widen/revert-2-fix-pk
jlloyd-widen Mar 24, 2022
f0679f1
Revert "attempted fix for hierarchy stream primary key"
jlloyd-widen Mar 24, 2022
ea6e377
Merge pull request #4 from Widen/revert-1-fix-pk
jlloyd-widen Mar 24, 2022
49e3187
Merge branch 'AutoIDM:main' into main
jlloyd-widen Mar 24, 2022
3b8c2bd
fixed primary keys
Mar 24, 2022
e23ef9e
Merge pull request #5 from Widen/fix-pk-with-config
jlloyd-widen Mar 24, 2022
824bb90
switch pk for location based streams
Mar 24, 2022
1680a45
removed unecessary select in meltano.yml
Mar 24, 2022
2bdc9a3
Merged changes in
visch Apr 4, 2022
f559b58
Merge pull request #7 from AutoIDM/fix-pk-merged
jlloyd-widen Apr 4, 2022
357c0c7
Campaign label primary_key added
visch Apr 4, 2022
8c4b190
Added pk for campaign labels, and explained the why behind _sdc_prima…
visch Apr 4, 2022
b1c36cb
Ran black lint
visch Apr 4, 2022
a705e01
Missed the json schema add for campaign label
visch Apr 4, 2022
7324daf
Merge pull request #8 from AutoIDM/fix-pk-merged
jlloyd-widen Apr 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,3 +132,7 @@ dmypy.json

# Pyre type checker
.pyre/

# IDE
.idea/
.DS_Store
7 changes: 4 additions & 3 deletions tap_googleads/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from singer_sdk.exceptions import FatalAPIError, RetriableAPIError

from tap_googleads.auth import GoogleAdsAuthenticator
from tap_googleads.utils import replicate_pk_at_root


SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
Expand All @@ -23,6 +24,7 @@ class GoogleAdsStream(RESTStream):

records_jsonpath = "$[*]" # Or override `parse_response`.
next_page_token_jsonpath = "$.nextPageToken" # Or override `get_next_page_token`.
primary_keys_jsonpaths = None
_LOG_REQUEST_METRIC_URLS: bool = True

@property
Expand Down Expand Up @@ -144,10 +146,9 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]:
# TODO: Parse response body and return a set of records.
yield from extract_jsonpath(self.records_jsonpath, input=response.json())

def post_process(self, row: dict, context: Optional[dict]) -> dict:
def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]:
"""As needed, append or transform raw data to match expected structure."""
# TODO: Delete this method if not needed.
return row
return replicate_pk_at_root(row, self.primary_keys_jsonpaths)


class CustomerNotEnabledError(Exception):
Expand Down
3 changes: 3 additions & 0 deletions tap_googleads/schemas/ad_group.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
{
"type": "object",
"properties": {
"_sdc_primary_key": {
"type": "string"
},
"adGroup": {
"type": "object",
"properties": {
Expand Down
14 changes: 11 additions & 3 deletions ...ogleads/schemas/adgroups_performance.json → ...ogleads/schemas/ad_group_performance.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
{
"type": "object",
"properties": {
"customer_id": {
"type": "string"
},
"campaign": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -39,6 +36,17 @@
"type": "string"
}
}
},
"segments": {
"type": "object",
"properties": {
"date": {
"type": "string"
}
}
},
"_sdc_primary_key": {
"type": "string"
}
}
}
6 changes: 3 additions & 3 deletions tap_googleads/schemas/campaign.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"type": "object",
"properties": {
"customer_id": {
"type": "string"
},
"_sdc_primary_key": {
"type": "string"
},
"campaign": {
"type": "object",
"properties": {
Expand Down
96 changes: 48 additions & 48 deletions tap_googleads/schemas/campaign_performance.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,53 +1,53 @@
{
"type": "object",
"properties": {
"customer_id": {
"type": "object",
"properties": {
"_sdc_primary_key": {
"type": "string"
},
"campaign": {
"type": "object",
"properties": {
"resourceName": {
"type": "string"
},
"campaign": {
"type": "object",
"properties": {
"resourceName": {
"type": "string"
},
"status": {
"type": "string"
},
"name": {
"type": "string"
}
}
},
"metrics": {
"type": "object",
"properties": {
"clicks": {
"type": "string"
},
"costMicros": {
"type": "string"
},
"ctr": {
"type": "number"
},
"averageCpc": {
"type": "number"
},
"impressions": {
"type": "string"
}
}
},
"segments": {
"type": "object",
"properties": {
"device": {
"type": "string"
},
"date": {
"type": "string"
}
}
}
"status": {
"type": "string"
},
"name": {
"type": "string"
}
}
},
"metrics": {
"type": "object",
"properties": {
"clicks": {
"type": "string"
},
"costMicros": {
"type": "string"
},
"ctr": {
"type": "number"
},
"averageCpc": {
"type": "number"
},
"impressions": {
"type": "string"
}
}
},
"segments": {
"type": "object",
"properties": {
"device": {
"type": "string"
},
"date": {
"type": "string"
}
}
}
}
}
168 changes: 84 additions & 84 deletions tap_googleads/schemas/campaign_performance_by_age_range_and_device.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,91 +1,91 @@
{
"type": "object",
"properties": {
"customer_id": {
"type": "object",
"properties": {
"_sdc_primary_key": {
"type": "string"
},
"campaign": {
"type": "object",
"properties": {
"resourceName": {
"type": "string"
},
"campaign": {
"type": "object",
"properties": {
"resourceName": {
"type": "string"
},
"status": {
"type": "string"
},
"advertisingChannelType": {
"type": "string"
},
"name": {
"type": "string"
}
}
},
"adGroup": {
"type": "object",
"properties": {
"resourceName": {
"type": "string"
},
"name": {
"type": "string"
}
}
},
"metrics": {
"type": "object",
"properties": {
"clicks": {
"type": "string"
},
"costMicros": {
"type": "string"
},
"ctr": {
"type": "number"
},
"averageCpc": {
"type": "number"
},
"impressions": {
"type": "string"
}
}
},
"adGroupCriterion": {
"type": "object",
"properties": {
"resourceName": {
"type": "string"
},
"ageRange": {
"type": "object",
"properties": {
"type": {
"type": "string"
}
}
}
}
},
"ageRangeView": {
"type": "object",
"properties": {
"resourceName": {
"type": "string"
}
}
},
"segments": {
"type": "object",
"properties": {
"device": {
"type": "string"
},
"date": {
"type": "string"
}
"status": {
"type": "string"
},
"advertisingChannelType": {
"type": "string"
},
"name": {
"type": "string"
}
}
},
"adGroup": {
"type": "object",
"properties": {
"resourceName": {
"type": "string"
},
"name": {
"type": "string"
}
}
},
"metrics": {
"type": "object",
"properties": {
"clicks": {
"type": "string"
},
"costMicros": {
"type": "string"
},
"ctr": {
"type": "number"
},
"averageCpc": {
"type": "number"
},
"impressions": {
"type": "string"
}
}
},
"adGroupCriterion": {
"type": "object",
"properties": {
"resourceName": {
"type": "string"
},
"ageRange": {
"type": "object",
"properties": {
"type": {
"type": "string"
}
}
}
}
},
"ageRangeView": {
"type": "object",
"properties": {
"resourceName": {
"type": "string"
}
}
},
"segments": {
"type": "object",
"properties": {
"device": {
"type": "string"
},
"date": {
"type": "string"
}
}
}
}
}
Loading