From 4adc66efd7a46bcff7e2674ab98af7313c2391d5 Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Mon, 11 Oct 2021 18:49:33 +0300 Subject: [PATCH 01/11] Source Hubspot: Fix issue with getting 414 HTTP error for streams --- .../connectors/source-hubspot/Dockerfile | 2 +- .../source-hubspot/acceptance-test-config.yml | 18 +- .../configured_catalog_without_workflows.json | 184 ------------------ .../sample_files/full_refresh_catalog.json | 134 +++++++++++++ .../source-hubspot/source_hubspot/api.py | 119 ++++++----- docs/integrations/sources/hubspot.md | 1 + 6 files changed, 214 insertions(+), 244 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_without_workflows.json create mode 100644 airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 6de5c2f3330dc..90896aec09d2a 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -14,5 +14,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh" -LABEL io.airbyte.version=0.1.16 +LABEL io.airbyte.version=0.1.17 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml index a47ab37f61986..9861bb055c244 100644 --- a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml @@ -17,20 +17,8 @@ tests: - config_path: "secrets/config.json" basic_read: - config_path: "secrets/config.json" - configured_catalog_path: "sample_files/configured_catalog.json" - empty_streams: - [ - "contact_lists", - "campaigns", - "tickets", - "subscription_changes", - "quotes", - "email_events", - "engagements", - "forms", - "products", - "workflows", - ] + configured_catalog_path: "sample_files/full_refresh_catalog.json" + empty_streams: ["workflows"] - config_path: "secrets/config_oauth.json" configured_catalog_path: "sample_files/configured_catalog.json" empty_streams: @@ -59,6 +47,6 @@ tests: # email_events: ["timestamp"] full_refresh: - config_path: "secrets/config.json" - configured_catalog_path: "sample_files/configured_catalog.json" + configured_catalog_path: "sample_files/full_refresh_catalog.json" - config_path: "secrets/config_oauth.json" configured_catalog_path: "sample_files/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_without_workflows.json b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_without_workflows.json deleted file mode 100644 index 06495e01ee0fd..0000000000000 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_without_workflows.json +++ /dev/null @@ -1,184 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "campaigns", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": ["lastUpdatedTime"] - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "companies", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": ["updatedAt"] - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "contact_lists", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": ["updatedAt"] - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "contacts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": ["updatedAt"] - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "deal_pipelines", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": ["updatedAt"] - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "deals", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": ["updatedAt"] - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "email_events", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"] - }, - "sync_mode": "incremental", - "cursor_field": ["created"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "engagements", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": ["lastUpdated"] - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "forms", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": ["updatedAt"] - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "line_items", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": ["updatedAt"] - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "owners", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": ["updatedAt"] - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "products", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": ["updatedAt"] - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "quotes", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": ["updatedAt"] - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "subscription_changes", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["timestamp"] - }, - "sync_mode": "incremental", - "cursor_field": ["timestamp"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "tickets", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": ["updatedAt"] - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json new file mode 100644 index 0000000000000..9b96706690e33 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json @@ -0,0 +1,134 @@ +{ + "streams": [ + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "companies", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "contact_lists", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "contacts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "deal_pipelines", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "deals", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "engagements", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "forms", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "line_items", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "owners", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "products", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "quotes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "tickets", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "workflows", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "default_cursor_field": ["updatedAt"] + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py index 071f103aaff2f..cae18eef55816 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py @@ -177,6 +177,7 @@ class Stream(ABC): page_field = "offset" limit_field = "limit" limit = 100 + offset = 0 @property @abstractmethod @@ -302,59 +303,89 @@ def _filter_old_records(self, records: Iterable) -> Iterable: yield record def _read(self, getter: Callable, params: MutableMapping[str, Any] = None) -> Iterator: + next_page_token = None while True: - response = getter(params=params) - if isinstance(response, Mapping): - if response.get("status", None) == "error": - """ - When the API Key doen't have the permissions to access the endpoint, - we break the read, skip this stream and log warning message for the user. - - Example: - - response.json() = { - 'status': 'error', - 'message': 'This hapikey (....) does not have proper permissions! (requires any of [automation-access])', - 'correlationId': '111111-2222-3333-4444-55555555555'} - """ - logger.warn(f"Stream `{self.data_field}` cannot be procced. {response.get('message')}") - break - - if response.get(self.data_field) is None: - """ - When the response doen't have the stream's data, raise an exception. - """ - raise RuntimeError("Unexpected API response: {} not in {}".format(self.data_field, response.keys())) - - yield from response[self.data_field] - - # pagination - if "paging" in response: # APIv3 pagination - if "next" in response["paging"]: - params["after"] = response["paging"]["next"]["after"] + if next_page_token: + params.update(next_page_token) + + properties_list = list(self.properties.keys()) + if properties_list: + # TODO: Additional processing was added due to the fact that users receive 414 errors while syncing their streams (issues #3977 and #5835). + # Unfortunately, this implementation is not 100% reliable, but there is no other alternative at the moment. + # We will need to fix this code when the Hubspot developers add the ability to use a special parameter to get all properties for an entity. + # According to Hubspot Community (https://community.hubspot.com/t5/APIs-Integrations/Get-all-contact-properties-without-explicitly-listing-them/m-p/447950) + # and the official documentation, this does not exist at the moment. + properties_length = 500 + stream_records = [] + + for property_index in range(0, len(properties_list), properties_length): + params.update({"properties": ",".join(properties_list[property_index : property_index + properties_length])}) + response = getter(params=params) + if stream_records: + for counter, record in enumerate(self.parse_response(response)): + if counter <= len(stream_records) and stream_records[counter].get("properties"): + stream_records[counter]["properties"].update(record.get("properties")) else: - break - else: - if not response.get(self.more_key, False): - break - if self.page_field in response: - params[self.page_filter] = response[self.page_field] + stream_records = list(self.parse_response(response)) + + yield from stream_records else: - response = list(response) - yield from response + response = getter(params=params) + yield from self.parse_response(response) - # pagination - if len(response) < self.limit: - break - else: - params[self.page_filter] = params.get(self.page_filter, 0) + self.limit + next_page_token = self.next_page_token(response) + if not next_page_token: + break def read(self, getter: Callable, params: Mapping[str, Any] = None) -> Iterator: - default_params = {self.limit_field: self.limit, "properties": ",".join(self.properties.keys())} + default_params = {self.limit_field: self.limit} params = {**default_params, **params} if params else {**default_params} - yield from self._filter_dynamic_fields(self._filter_old_records(self._transform(self._read(getter, params)))) + def parse_response(self, response: Union[Mapping[str, Any], List[dict]]) -> Iterator: + if isinstance(response, Mapping): + if response.get("status", None) == "error": + """ + When the API Key doen't have the permissions to access the endpoint, + we break the read, skip this stream and log warning message for the user. + + Example: + + response.json() = { + 'status': 'error', + 'message': 'This hapikey (....) does not have proper permissions! (requires any of [automation-access])', + 'correlationId': '111111-2222-3333-4444-55555555555'} + """ + logger.warn(f"Stream `{self.data_field}` cannot be procced. {response.get('message')}") + return + + if response.get(self.data_field) is None: + """ + When the response doen't have the stream's data, raise an exception. + """ + raise RuntimeError("Unexpected API response: {} not in {}".format(self.data_field, response.keys())) + + yield from response[self.data_field] + + else: + response = list(response) + yield from response + + def next_page_token(self, response: Union[Mapping[str, Any], List[dict]]) -> Optional[Mapping[str, Union[int, str]]]: + if isinstance(response, Mapping): + if "paging" in response: # APIv3 pagination + if "next" in response["paging"]: + return {"after": response["paging"]["next"]["after"]} + else: + if not response.get(self.more_key, False): + return + if self.page_field in response: + return {self.page_filter: response[self.page_field]} + else: + if len(response) >= self.limit: + self.offset += self.limit + return {self.page_filter: self.offset} + @staticmethod def _get_field_props(field_type: str) -> Mapping[str, List[str]]: diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 092f9c34237b2..3ae2679f4c9c5 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -93,6 +93,7 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.17 | 2021-10-?? | [????](https://github.com/airbytehq/airbyte/pull/????) | Fix issue with getting 414 HTTP error for streams | | 0.1.16 | 2021-09-27 | [6465](https://github.com/airbytehq/airbyte/pull/6465) | Implement OAuth support. Use CDK authenticator instead of connector specific authenticator | | 0.1.15 | 2021-09-23 | [6374](https://github.com/airbytehq/airbyte/pull/6374) | Use correct schema for `owners` stream | | 0.1.14 | 2021-09-08 | [5693](https://github.com/airbytehq/airbyte/pull/5693) | Include deal\_to\_contact association when pulling deal stream and include contact ID in contact stream | From 7e3f924164f1ce9cd2e75671b9fe35cbc3cb5b84 Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Mon, 11 Oct 2021 18:49:59 +0300 Subject: [PATCH 02/11] update docs --- docs/integrations/sources/hubspot.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 3ae2679f4c9c5..d69caa4be66b0 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -93,7 +93,7 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.17 | 2021-10-?? | [????](https://github.com/airbytehq/airbyte/pull/????) | Fix issue with getting 414 HTTP error for streams | +| 0.1.17 | 2021-10-?? | [????](https://github.com/airbytehq/airbyte/pull/????) | Fix issue with getting `414` HTTP error for streams | | 0.1.16 | 2021-09-27 | [6465](https://github.com/airbytehq/airbyte/pull/6465) | Implement OAuth support. Use CDK authenticator instead of connector specific authenticator | | 0.1.15 | 2021-09-23 | [6374](https://github.com/airbytehq/airbyte/pull/6374) | Use correct schema for `owners` stream | | 0.1.14 | 2021-09-08 | [5693](https://github.com/airbytehq/airbyte/pull/5693) | Include deal\_to\_contact association when pulling deal stream and include contact ID in contact stream | From 646a9126178fa1ea38cca970371a6bd9beebddda Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Mon, 11 Oct 2021 18:54:57 +0300 Subject: [PATCH 03/11] update credentials --- tools/bin/ci_credentials.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 34b7ed1cfb4bb..4db9977bca900 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -86,7 +86,7 @@ write_standard_creds source-greenhouse "$GREENHOUSE_TEST_CREDS" write_standard_creds source-greenhouse "$GREENHOUSE_TEST_CREDS_LIMITED" "config_users_only.json" write_standard_creds source-harvest "$HARVEST_INTEGRATION_TESTS_CREDS" write_standard_creds source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS" -write_standard_creds source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS" "config_oauth.json" +write_standard_creds source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH" "config_oauth.json" write_standard_creds source-instagram "$INSTAGRAM_INTEGRATION_TESTS_CREDS" write_standard_creds source-intercom "$INTERCOM_INTEGRATION_TEST_CREDS" write_standard_creds source-iterable "$ITERABLE_INTEGRATION_TEST_CREDS" From bb2d4055a28fc669f42e31c148bdc6f70386a45e Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Tue, 12 Oct 2021 14:20:08 +0300 Subject: [PATCH 04/11] little fix --- .../connectors/source-hubspot/source_hubspot/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py index cae18eef55816..17e3b6afa6cd8 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py @@ -324,7 +324,7 @@ def _read(self, getter: Callable, params: MutableMapping[str, Any] = None) -> It if stream_records: for counter, record in enumerate(self.parse_response(response)): if counter <= len(stream_records) and stream_records[counter].get("properties"): - stream_records[counter]["properties"].update(record.get("properties")) + stream_records[counter]["properties"].update(record.get("properties", {})) else: stream_records = list(self.parse_response(response)) From 99695d05e367959dfdf98dfa2d4015ca2079286a Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Wed, 13 Oct 2021 14:35:51 +0300 Subject: [PATCH 05/11] a little update --- .../connectors/source-hubspot/source_hubspot/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py index 17e3b6afa6cd8..bf49bb4b18eed 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py @@ -356,7 +356,7 @@ def parse_response(self, response: Union[Mapping[str, Any], List[dict]]) -> Iter 'message': 'This hapikey (....) does not have proper permissions! (requires any of [automation-access])', 'correlationId': '111111-2222-3333-4444-55555555555'} """ - logger.warn(f"Stream `{self.data_field}` cannot be procced. {response.get('message')}") + logger.warn(f"Stream `{self.entity}` cannot be procced. {response.get('message')}") return if response.get(self.data_field) is None: From 671bb3abff2d34d772221dd1d89940d720cd25d1 Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Mon, 18 Oct 2021 17:52:19 +0300 Subject: [PATCH 06/11] update code and schemas --- .../source-hubspot/source_hubspot/api.py | 42 +- .../source_hubspot/schemas/companies.json | 64 +- .../source_hubspot/schemas/contacts.json | 2162 +---------------- .../source_hubspot/schemas/forms.json | 296 +-- .../source_hubspot/schemas/line_items.json | 20 +- .../source_hubspot/schemas/products.json | 20 +- .../source_hubspot/schemas/quotes.json | 9 +- .../source_hubspot/schemas/tickets.json | 20 +- 8 files changed, 197 insertions(+), 2436 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py index bf49bb4b18eed..ae4f291f1e934 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py @@ -8,7 +8,7 @@ from abc import ABC, abstractmethod from functools import lru_cache, partial from http import HTTPStatus -from typing import Any, Callable, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Union +from typing import Any, Callable, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union import backoff import pendulum as pendulum @@ -17,6 +17,8 @@ from base_python.entrypoint import logger from source_hubspot.errors import HubspotAccessDenied, HubspotInvalidAuth, HubspotRateLimited, HubspotTimeout +PROPERTIES_PARAM_MAX_LENGTH = 1500 + # we got this when provided API Token has incorrect format CLOUDFLARE_ORIGIN_DNS_ERROR = 530 @@ -49,6 +51,22 @@ CUSTOM_FIELD_VALUE_TO_TYPE = {v: k for k, v in CUSTOM_FIELD_TYPE_TO_VALUE.items()} +def split_properties(properties_list: List[str]) -> Iterator[Tuple[str]]: + summary_length = 0 + local_properties = [] + for property_ in properties_list: + if len(property_) + summary_length >= PROPERTIES_PARAM_MAX_LENGTH: + yield local_properties + local_properties = [] + summary_length = 0 + + local_properties.append(property_) + summary_length += len(property_) + + if local_properties: + yield local_properties + + def retry_connection_handler(**kwargs): """Retry helper, log each attempt""" @@ -315,23 +333,23 @@ def _read(self, getter: Callable, params: MutableMapping[str, Any] = None) -> It # We will need to fix this code when the Hubspot developers add the ability to use a special parameter to get all properties for an entity. # According to Hubspot Community (https://community.hubspot.com/t5/APIs-Integrations/Get-all-contact-properties-without-explicitly-listing-them/m-p/447950) # and the official documentation, this does not exist at the moment. - properties_length = 500 stream_records = [] - for property_index in range(0, len(properties_list), properties_length): - params.update({"properties": ",".join(properties_list[property_index : property_index + properties_length])}) + for properties in split_properties(properties_list): + params.update({"properties": ",".join(properties)}) response = getter(params=params) if stream_records: - for counter, record in enumerate(self.parse_response(response)): - if counter <= len(stream_records) and stream_records[counter].get("properties"): - stream_records[counter]["properties"].update(record.get("properties", {})) + for record in self._transform(self.parse_response(response)): + index = next((i for i, item in enumerate(stream_records) if item.get("id") == record.get("id")), -1) + if index != -1 and stream_records[index].get("properties"): + stream_records[index]["properties"].update(record.get("properties", {})) else: - stream_records = list(self.parse_response(response)) + stream_records = list(self._transform(self.parse_response(response))) yield from stream_records else: response = getter(params=params) - yield from self.parse_response(response) + yield from self._transform(self.parse_response(response)) next_page_token = self.next_page_token(response) if not next_page_token: @@ -340,7 +358,7 @@ def _read(self, getter: Callable, params: MutableMapping[str, Any] = None) -> It def read(self, getter: Callable, params: Mapping[str, Any] = None) -> Iterator: default_params = {self.limit_field: self.limit} params = {**default_params, **params} if params else {**default_params} - yield from self._filter_dynamic_fields(self._filter_old_records(self._transform(self._read(getter, params)))) + yield from self._filter_dynamic_fields(self._filter_old_records(self._read(getter, params))) def parse_response(self, response: Union[Mapping[str, Any], List[dict]]) -> Iterator: if isinstance(response, Mapping): @@ -670,13 +688,13 @@ def _transform(self, records: Iterable) -> Iterable: class FormStream(Stream): - """Marketing Forms, API v2 + """Marketing Forms, API v3 by default non-marketing forms are filtered out of this endpoint Docs: https://developers.hubspot.com/docs/api/marketing/forms """ entity = "form" - url = "/forms/v2/forms" + url = "/marketing/v3/forms" updated_at_field = "updatedAt" created_at_field = "createdAt" diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies.json index eb20e11de8a95..ff5e4b359c9bd 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies.json @@ -2,67 +2,25 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], "properties": { - "portalId": { - "type": ["null", "integer"] - }, - "companyId": { - "type": ["null", "integer"] - }, - "isDeleted": { - "type": ["null", "boolean"] + "id": { + "type": ["null", "string"] }, - "stateChanges": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"] - } + "createdAt": { + "type": ["null", "string"], + "format": "date-time" }, - "additionalDomains": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" }, - "mergeAudits": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "mergedCompanyId": { - "type": ["null", "integer"] - }, - "canonicalCompanyId": { - "type": ["null", "integer"] - }, - "sourceId": { - "type": ["null", "string"] - }, - "entityId": { - "type": ["null", "string"] - }, - "mergedCompanyName": { - "type": ["null", "string"] - }, - "movedProperties": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - } - } - } + "archived": { + "type": ["null", "boolean"] }, "contacts": { "type": ["null", "array"], "items": { - "type": ["null", "string"] + "type": "string" } - }, - "createdAt": { - "type": ["null", "string"] - }, - "updatedAt": { - "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts.json index 5187593975f58..1459718fe8444 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts.json @@ -3,2166 +3,18 @@ "type": ["null", "object"], "properties": { "id": { - "type": "string" - }, - "vid": { - "type": ["null", "integer"] - }, - "canonical-vid": { - "type": ["null", "integer"] - }, - "merged-vids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "portal-id": { - "type": ["null", "integer"] - }, - "is-contact": { - "type": ["null", "boolean"] - }, - "profile-token": { - "type": ["null", "string"] - }, - "profile-url": { "type": ["null", "string"] }, - "associated-company": { - "type": ["null", "object"], - "properties": { - "properties": { - "type": ["null", "object"], - "properties": { - "about_us": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "closedate_timestamp_earliest_value_a2a17e6e": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "facebookfans": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "first_contact_createdate_timestamp_earliest_value_78b50eea": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "first_conversion_date_timestamp_earliest_value_61f58f2c": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "first_conversion_event_name_timestamp_earliest_value_68ddae0a": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "first_deal_created_date": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "founded_year": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_additional_domains": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_timestamp": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_touch_converting_campaign": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_visit_timestamp": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_timestamp": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_touch_converting_campaign": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_visit_timestamp": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_num_page_views": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_num_page_views_cardinality_sum_e46e85b0": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_num_visits": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_num_visits_cardinality_sum_53d952a6": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source_data_1": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source_data_2": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source_timestamp_earliest_value_25a3a52c": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_avatar_filemanager_key": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_created_by_user_id": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_createdate": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_ideal_customer_profile": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_is_target_account": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "boolean"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_last_booked_meeting_date": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_last_logged_call_date": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_last_open_task_date": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_last_sales_activity_date": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_last_sales_activity_timestamp": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_lastmodifieddate": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_merged_object_ids": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_num_blockers": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_num_contacts_with_buying_roles": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_num_decision_makers": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_num_open_deals": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_object_id": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_target_account": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_target_account_probability": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_target_account_recommendation_snooze_time": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_target_account_recommendation_state": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_total_deal_value": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_updated_by_user_id": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_user_ids_of_all_owners": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hubspot_owner_assigneddate": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "is_public": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "boolean"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "num_associated_contacts": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "num_associated_deals": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "num_conversion_events_cardinality_sum_d095f14b": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "recent_conversion_date_timestamp_latest_value_72856da1": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "recent_conversion_event_name_timestamp_latest_value_66c820bf": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "recent_deal_amount": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "recent_deal_close_date": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "timezone": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "total_money_raised": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "total_revenue": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "twitterhandle": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "phone": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "twitterbio": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "twitterfollowers": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "address": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "address2": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "facebook_company_page": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "city": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "linkedin_company_page": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "linkedinbio": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "state": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "googleplus_page": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "engagements_last_meeting_booked": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "engagements_last_meeting_booked_campaign": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "engagements_last_meeting_booked_medium": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "engagements_last_meeting_booked_source": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_latest_meeting_activity": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_sales_email_last_replied": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hubspot_owner_id": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "notes_last_contacted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "notes_last_updated": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "notes_next_activity_date": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "num_contacted_notes": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "num_notes": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "zip": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "country": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hubspot_team_id": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_all_owner_ids": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "website": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "domain": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_all_team_ids": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_all_accessible_team_ids": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "numberofemployees": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "industry": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "annualrevenue": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "lifecyclestage": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_lead_status": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_parent_company_id": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "type": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "description": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_num_child_companies": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "createdate": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "closedate": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "first_contact_createdate": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "days_to_close": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "web_technologies": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - } - } - }, - "company-id": { - "type": ["null", "integer"] - }, - "portal-id": { - "type": ["null", "integer"] - } - } - }, - "identity-profiles": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "deleted-changed-timestamp": { - "type": ["null", "string"] - }, - "saved-at-timestamp": { - "type": ["null", "string"] - }, - "vid": { - "type": ["null", "integer"] - }, - "identities": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "timestamp": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "list-memberships": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "internal-list-id": { - "type": ["null", "integer"] - }, - "is-member": { - "type": ["null", "boolean"] - }, - "static-list-id": { - "type": ["null", "integer"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "vid": { - "type": ["null", "integer"] - } - } - } - }, - "form-submissions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "conversion-id": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "form-id": { - "type": ["null", "string"] - }, - "portal-id": { - "type": ["null", "integer"] - }, - "page-url": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - } - } - } - }, - "merge-audits": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "canonical-vid": { - "type": ["null", "integer"] - }, - "vid-to-merge": { - "type": ["null", "integer"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "user-id": { - "type": ["null", "integer"] - }, - "num-properties-moved": { - "type": ["null", "integer"] - }, - "merged_from_email": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "source-type": { - "type": ["null", "string"] - }, - "source-id": { - "type": ["null", "string"] - }, - "source-label": { - "type": ["null", "string"] - }, - "source-vids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "timestamp": { - "type": ["null", "integer"] - }, - "selected": { - "type": ["null", "boolean"] - } - } - }, - "merged_to_email": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "source-type": { - "type": ["null", "string"] - }, - "source-id": { - "type": ["null", "string"] - }, - "source-label": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "integer"] - }, - "selected": { - "type": ["null", "boolean"] - } - } - } - } - } - }, "createdAt": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "updatedAt": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" + }, + "archived": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json index b5f69999c3f44..a20d12b16e70d 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json @@ -2,43 +2,27 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], "properties": { - "deletedAt": { - "type": ["null", "integer"] - }, - "portalId": { - "type": ["null", "integer"] - }, - "guid": { + "id": { "type": ["null", "string"] }, "name": { "type": ["null", "string"] }, - "action": { - "type": ["null", "string"] - }, - "method": { - "type": ["null", "string"] - }, - "cssClass": { - "type": ["null", "string"] - }, - "redirect": { - "type": ["null", "string"] - }, - "submitText": { - "type": ["null", "string"] + "createdAt": { + "type": ["null", "integer"], + "format": "date-time" }, - "followUpId": { - "type": ["null", "string"] + "updatedAt": { + "type": ["null", "integer"], + "format": "date-time" }, - "notifyRecipients": { - "type": ["null", "string"] + "archived": { + "type": ["null", "boolean"] }, - "leadNurturingCampaignId": { - "type": ["null", "string"] + "deletedAt": { + "type": ["null", "integer"] }, - "formFieldGroups": { + "fieldGroups": { "type": ["null", "array"], "items": { "type": ["null", "object"], @@ -48,181 +32,139 @@ "items": { "type": ["null", "object"], "properties": { - "name": { + "objectTypeId": { "type": ["null", "string"] }, - "label": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "fieldType": { - "type": ["null", "string"] - }, - "description": { + "name": { "type": ["null", "string"] }, - "groupName": { + "label": { "type": ["null", "string"] }, - "displayOrder": { - "type": ["null", "integer"] - }, "required": { "type": ["null", "boolean"] }, - "validation": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "message": { - "type": ["null", "string"] - }, - "data": { - "type": ["null", "string"] - }, - "useDefaultBlockList": { - "type": ["null", "boolean"] - }, - "blockedEmailAddresses": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - } - } - }, - "enabled": { - "type": ["null", "boolean"] - }, "hidden": { "type": ["null", "boolean"] }, - "defaultValue": { - "type": ["null", "string"] - }, - "isSmartField": { - "type": ["null", "boolean"] - }, - "unselectedLabel": { - "type": ["null", "string"] - }, - "placeholder": { + "fieldType": { "type": ["null", "string"] - }, - "labelHidden": { - "type": ["null", "boolean"] - }, - "options": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "description": { - "type": ["null", "string"] - }, - "displayOrder": { - "type": ["null", "integer"] - }, - "doubleData": { - "type": ["null", "number"] - }, - "hidden": { - "type": ["null", "boolean"] - }, - "label": { - "type": ["null", "string"] - }, - "readOnly": { - "type": ["null", "boolean"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "selectedOptions": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } } } } }, - "default": { - "type": ["null", "boolean"] - }, - "isSmartGroup": { - "type": ["null", "boolean"] + "groupType": { + "type": ["null", "string"] }, - "richText": { - "type": ["null", "object"], - "properties": { - "content": { - "type": ["null", "string"] - } - } + "richTextType": { + "type": ["null", "string"] } } } }, - "createdAt": { - "type": ["null", "integer"] - }, - "updatedAt": { - "type": ["null", "integer"] - }, - "performableHtml": { - "type": ["null", "string"] - }, - "migratedFrom": { - "type": ["null", "string"] - }, - "ignoreCurrentValues": { - "type": ["null", "boolean"] - }, - "deletable": { - "type": ["null", "boolean"] - }, - "inlineMessage": { - "type": ["null", "string"] - }, - "tmsId": { - "type": ["null", "string"] - }, - "captchaEnabled": { - "type": ["null", "boolean"] - }, - "campaignGuid": { - "type": ["null", "string"] - }, - "cloneable": { - "type": ["null", "boolean"] - }, - "editable": { - "type": ["null", "boolean"] - }, - "formType": { - "type": ["null", "string"] + "configuration": { + "type": ["null", "object"], + "properties": { + "language": { + "type": ["null", "string"] + }, + "cloneable": { + "type": ["null", "boolean"] + }, + "postSubmitAction": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + }, + "editable": { + "type": ["null", "boolean"] + }, + "archivable": { + "type": ["null", "boolean"] + }, + "recaptchaEnabled": { + "type": ["null", "boolean"] + }, + "notifyContactOwner": { + "type": ["null", "boolean"] + }, + "notifyRecipients": { + "type": ["null", "array"] + }, + "createNewContactForNewEmail": { + "type": ["null", "boolean"] + }, + "prePopulateKnownValues": { + "type": ["null", "boolean"] + }, + "allowLinkToResetKnownValues": { + "type": ["null", "boolean"] + } + } }, - "metaData": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] + "displayOptions": { + "type": ["null", "object"], + "properties": { + "renderRawHtml": { + "type": ["null", "boolean"] + }, + "theme": { + "type": ["null", "string"] + }, + "submitButtonText": { + "type": ["null", "string"] + }, + "style": { + "type": ["null", "object"], + "properties": { + "fontFamily": { + "type": ["null", "string"] + }, + "backgroundWidth": { + "type": ["null", "string"] + }, + "labelTextColor": { + "type": ["null", "string"] + }, + "labelTextSize": { + "type": ["null", "string"] + }, + "helpTextColor": { + "type": ["null", "string"] + }, + "helpTextSize": { + "type": ["null", "string"] + }, + "legalConsentTextColor": { + "type": ["null", "string"] + }, + "legalConsentTextSize": { + "type": ["null", "string"] + }, + "submitColor": { + "type": ["null", "string"] + }, + "submitAlignment": { + "type": ["null", "string"] + }, + "submitFontColor": { + "type": ["null", "string"] + }, + "submitSize": { + "type": ["null", "string"] + } } } } + }, + "legalConsentOptions": { + "type": ["null", "object"] } } } diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/line_items.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/line_items.json index ad845580d83ac..1459718fe8444 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/line_items.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/line_items.json @@ -2,23 +2,19 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], "properties": { - "objectType": { + "id": { "type": ["null", "string"] }, - "portalId": { - "type": ["null", "integer"] - }, - "objectId": { - "type": ["null", "integer"] - }, - "isDeleted": { - "type": ["null", "boolean"] - }, "createdAt": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "updatedAt": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" + }, + "archived": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/products.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/products.json index ad845580d83ac..1459718fe8444 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/products.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/products.json @@ -2,23 +2,19 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], "properties": { - "objectType": { + "id": { "type": ["null", "string"] }, - "portalId": { - "type": ["null", "integer"] - }, - "objectId": { - "type": ["null", "integer"] - }, - "isDeleted": { - "type": ["null", "boolean"] - }, "createdAt": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "updatedAt": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" + }, + "archived": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/quotes.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/quotes.json index 63cedd803b129..2c78c1b8a97e3 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/quotes.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/quotes.json @@ -6,16 +6,19 @@ "type": ["null", "string"] }, "createdAt": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "updatedAt": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "archived": { "type": ["null", "boolean"] }, "archivedAt": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "associations": { "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/tickets.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/tickets.json index ad845580d83ac..1459718fe8444 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/tickets.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/tickets.json @@ -2,23 +2,19 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], "properties": { - "objectType": { + "id": { "type": ["null", "string"] }, - "portalId": { - "type": ["null", "integer"] - }, - "objectId": { - "type": ["null", "integer"] - }, - "isDeleted": { - "type": ["null", "boolean"] - }, "createdAt": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "updatedAt": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" + }, + "archived": { + "type": ["null", "boolean"] } } } From 6b07b552963d4ba25c9e7f4aa652a9d45069c754 Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Tue, 19 Oct 2021 00:14:00 +0300 Subject: [PATCH 07/11] add unit test --- .../source-hubspot/source_hubspot/api.py | 2 +- .../source-hubspot/unit_tests/test_client.py | 68 ++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py index ae4f291f1e934..87c1ed90bc3cc 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py @@ -17,7 +17,7 @@ from base_python.entrypoint import logger from source_hubspot.errors import HubspotAccessDenied, HubspotInvalidAuth, HubspotRateLimited, HubspotTimeout -PROPERTIES_PARAM_MAX_LENGTH = 1500 +PROPERTIES_PARAM_MAX_LENGTH = 15000 # we got this when provided API Token has incorrect format CLOUDFLARE_ORIGIN_DNS_ERROR = 530 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py index 2b5e6e360fca2..559759d96d123 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py @@ -4,7 +4,7 @@ import pytest -from source_hubspot.api import API +from source_hubspot.api import API, split_properties from source_hubspot.client import Client @@ -101,3 +101,69 @@ def get(url=test_stream.url, params=None): # match logged expected logged warning message with output given from preudo-output assert expected_warining_message + + +def test_splitting_properties(requests_mock, some_credentials): + """ + Check working stream `companies` with large list of properties using new functionality with splitting properties + """ + # Define stream name + stream_name = "companies" + number_of_properties = 2000 + + client = Client(start_date="2021-02-01T00:00:00Z", credentials=some_credentials) + api = API(some_credentials) + + properties_list = [f"property_number_{i}" for i in range(number_of_properties)] + parsed_properties = list(split_properties(properties_list)) + + # Check that properties are split into multiple arrays + assert len(parsed_properties) > 1 + + properties_response = [ + { + "json": [ + {"name": property_name, "type": "string", "updatedAt": 1571085954360, "createdAt": 1565059306048} + for property_name in properties_list + ], + "status_code": 200, + }, + ] + requests_mock.register_uri("GET", "/properties/v2/company/properties", properties_response) + + # Create test_stream instance + test_stream = client._apis.get(stream_name) + + for property_slice in parsed_properties: + record_responses = [ + { + "json": { + "results": [ + { + "id": id, + "properties": {p: "fake_data" for p in property_slice}, + "createdAt": "2020-12-10T07:58:09.554Z", + "updatedAt": "2021-07-31T08:18:58.954Z", + "archived": False, + } + for id in ["6043593519", "1092593519", "1092593518", "1092593517", "1092593516"] + ], + "paging": {}, + }, + "status_code": 200, + } + ] + requests_mock.register_uri("GET", f"{test_stream.url}?properties={','.join(property_slice)}", record_responses) + + # Mock the getter method that handles requests. + def get(url=test_stream.url, params=None): + response = api._session.get(api.BASE_URL + url, params=params) + return api._parse_and_handle_errors(response) + + # Read preudo-output from generator object read(), based on real scenario + stream_records = list(test_stream.read(getter=get)) + + # check that we have records for all set ids, and that each record has 2000 properties (not more, and not less) + assert len(stream_records) == 5 + for record in stream_records: + assert len(record["properties"]) == number_of_properties From eeb78331e54d4beae075ed1621f7020fcfce0bb2 Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Tue, 19 Oct 2021 00:25:47 +0300 Subject: [PATCH 08/11] update fields type for forms schema --- .../source-hubspot/source_hubspot/schemas/forms.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json index a20d12b16e70d..de55f229a1c7f 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json @@ -9,18 +9,18 @@ "type": ["null", "string"] }, "createdAt": { - "type": ["null", "integer"], + "type": ["null", "string"], "format": "date-time" }, "updatedAt": { - "type": ["null", "integer"], + "type": ["null", "string"], "format": "date-time" }, "archived": { "type": ["null", "boolean"] }, "deletedAt": { - "type": ["null", "integer"] + "type": ["null", "string"] }, "fieldGroups": { "type": ["null", "array"], From f95d95fbab9bb072d6dcfd5fc467d6768b8d0cd2 Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Thu, 21 Oct 2021 14:19:11 +0300 Subject: [PATCH 09/11] update code after review, add new tests --- .../source-hubspot/source_hubspot/api.py | 21 +- .../source-hubspot/unit_tests/test_client.py | 255 ++++++++++++------ 2 files changed, 184 insertions(+), 92 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py index 87c1ed90bc3cc..4d61ee5808f58 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py @@ -17,6 +17,8 @@ from base_python.entrypoint import logger from source_hubspot.errors import HubspotAccessDenied, HubspotInvalidAuth, HubspotRateLimited, HubspotTimeout +# The value is obtained experimentally, Hubspot allows the URL length up to ~ 16300 symbols, +# so it was decided to limit the length of the `properties` parameter to 15000 characters. PROPERTIES_PARAM_MAX_LENGTH = 15000 # we got this when provided API Token has incorrect format @@ -329,24 +331,21 @@ def _read(self, getter: Callable, params: MutableMapping[str, Any] = None) -> It properties_list = list(self.properties.keys()) if properties_list: # TODO: Additional processing was added due to the fact that users receive 414 errors while syncing their streams (issues #3977 and #5835). - # Unfortunately, this implementation is not 100% reliable, but there is no other alternative at the moment. # We will need to fix this code when the Hubspot developers add the ability to use a special parameter to get all properties for an entity. # According to Hubspot Community (https://community.hubspot.com/t5/APIs-Integrations/Get-all-contact-properties-without-explicitly-listing-them/m-p/447950) # and the official documentation, this does not exist at the moment. - stream_records = [] + stream_records = {} for properties in split_properties(properties_list): params.update({"properties": ",".join(properties)}) response = getter(params=params) - if stream_records: - for record in self._transform(self.parse_response(response)): - index = next((i for i, item in enumerate(stream_records) if item.get("id") == record.get("id")), -1) - if index != -1 and stream_records[index].get("properties"): - stream_records[index]["properties"].update(record.get("properties", {})) - else: - stream_records = list(self._transform(self.parse_response(response))) - - yield from stream_records + for record in self._transform(self.parse_response(response)): + if record["id"] not in stream_records: + stream_records[record["id"]] = record + elif stream_records[record["id"]].get("properties"): + stream_records[record["id"]]["properties"].update(record.get("properties", {})) + + yield from [value for key, value in stream_records.items()] else: response = getter(params=params) yield from self._transform(self.parse_response(response)) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py index 559759d96d123..e67e27f221ae1 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py @@ -3,10 +3,14 @@ # +from functools import partial + import pytest -from source_hubspot.api import API, split_properties +from source_hubspot.api import API, PROPERTIES_PARAM_MAX_LENGTH, split_properties from source_hubspot.client import Client +NUMBER_OF_PROPERTIES = 2000 + @pytest.fixture(name="some_credentials") def some_credentials_fixture(): @@ -18,35 +22,40 @@ def creds_with_wrong_permissions(): return {"credentials_title": "API Key Credentials", "api_key": "THIS-IS-THE-API_KEY"} -def test_client_backoff_on_limit_reached(requests_mock, some_credentials): - """Error once, check that we retry and not fail""" - responses = [ - {"json": {"error": "limit reached"}, "status_code": 429, "headers": {"Retry-After": "0"}}, - {"json": [], "status_code": 200}, - ] - - requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) - client = Client(start_date="2021-02-01T00:00:00Z", credentials=some_credentials) - - alive, error = client.health_check() - - assert alive - assert not error - +@pytest.fixture(name="fake_properties_list") +def fake_properties_list(): + return [f"property_number_{i}" for i in range(NUMBER_OF_PROPERTIES)] -def test_client_backoff_on_server_error(requests_mock, some_credentials): - """Error once, check that we retry and not fail""" - responses = [ - {"json": {"error": "something bad"}, "status_code": 500}, - {"json": [], "status_code": 200}, - ] - requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) - client = Client(start_date="2021-02-01T00:00:00Z", credentials=some_credentials) - alive, error = client.health_check() - - assert alive - assert not error +# def test_client_backoff_on_limit_reached(requests_mock, some_credentials): +# """Error once, check that we retry and not fail""" +# responses = [ +# {"json": {"error": "limit reached"}, "status_code": 429, "headers": {"Retry-After": "0"}}, +# {"json": [], "status_code": 200}, +# ] +# +# requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) +# client = Client(start_date="2021-02-01T00:00:00Z", credentials=some_credentials) +# +# alive, error = client.health_check() +# +# assert alive +# assert not error + + +# def test_client_backoff_on_server_error(requests_mock, some_credentials): +# """Error once, check that we retry and not fail""" +# responses = [ +# {"json": {"error": "something bad"}, "status_code": 500}, +# {"json": [], "status_code": 200}, +# ] +# requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) +# client = Client(start_date="2021-02-01T00:00:00Z", credentials=some_credentials) +# +# alive, error = client.health_check() +# +# assert alive +# assert not error def test_wrong_permissions_api_key(requests_mock, creds_with_wrong_permissions): @@ -103,67 +112,151 @@ def get(url=test_stream.url, params=None): assert expected_warining_message -def test_splitting_properties(requests_mock, some_credentials): - """ - Check working stream `companies` with large list of properties using new functionality with splitting properties - """ - # Define stream name - stream_name = "companies" - number_of_properties = 2000 - - client = Client(start_date="2021-02-01T00:00:00Z", credentials=some_credentials) - api = API(some_credentials) +class TestSplittingPropertiesFunctionality: + BASE_OBJECT_BODY = { + "createdAt": "2020-12-10T07:58:09.554Z", + "updatedAt": "2021-07-31T08:18:58.954Z", + "archived": False, + } - properties_list = [f"property_number_{i}" for i in range(number_of_properties)] - parsed_properties = list(split_properties(properties_list)) + @pytest.fixture + def client(self, some_credentials): + return Client(start_date="2021-02-01T00:00:00Z", credentials=some_credentials) - # Check that properties are split into multiple arrays - assert len(parsed_properties) > 1 + @pytest.fixture + def api(self, some_credentials): + return API(some_credentials) - properties_response = [ - { - "json": [ - {"name": property_name, "type": "string", "updatedAt": 1571085954360, "createdAt": 1565059306048} - for property_name in properties_list - ], - "status_code": 200, - }, - ] - requests_mock.register_uri("GET", "/properties/v2/company/properties", properties_response) - - # Create test_stream instance - test_stream = client._apis.get(stream_name) - - for property_slice in parsed_properties: - record_responses = [ + @staticmethod + def set_mock_properties(requests_mock, url, fake_properties_list): + properties_response = [ { - "json": { - "results": [ - { - "id": id, - "properties": {p: "fake_data" for p in property_slice}, - "createdAt": "2020-12-10T07:58:09.554Z", - "updatedAt": "2021-07-31T08:18:58.954Z", - "archived": False, - } - for id in ["6043593519", "1092593519", "1092593518", "1092593517", "1092593516"] - ], - "paging": {}, - }, + "json": [ + {"name": property_name, "type": "string", "updatedAt": 1571085954360, "createdAt": 1565059306048} + for property_name in fake_properties_list + ], "status_code": 200, - } + }, ] - requests_mock.register_uri("GET", f"{test_stream.url}?properties={','.join(property_slice)}", record_responses) + requests_mock.register_uri("GET", url, properties_response) # Mock the getter method that handles requests. - def get(url=test_stream.url, params=None): + def get(self, url, api, params=None): response = api._session.get(api.BASE_URL + url, params=params) return api._parse_and_handle_errors(response) - # Read preudo-output from generator object read(), based on real scenario - stream_records = list(test_stream.read(getter=get)) - - # check that we have records for all set ids, and that each record has 2000 properties (not more, and not less) - assert len(stream_records) == 5 - for record in stream_records: - assert len(record["properties"]) == number_of_properties + def test_splitting_properties(self, fake_properties_list): + """ + Check that properties are split into multiple arrays + """ + for slice_property in split_properties(fake_properties_list): + slice_length = [len(item) for item in slice_property] + assert sum(slice_length) <= PROPERTIES_PARAM_MAX_LENGTH + + def test_stream_with_splitting_properties(self, requests_mock, client, api, fake_properties_list): + """ + Check working stream `companies` with large list of properties using new functionality with splitting properties + """ + # Define stream name + stream_name = "companies" + + parsed_properties = list(split_properties(fake_properties_list)) + self.set_mock_properties(requests_mock, "/properties/v2/company/properties", fake_properties_list) + + # Create test_stream instance + test_stream = client._apis.get(stream_name) + record_ids_paginated = [list(map(str, range(100))), list(map(str, range(100, 150, 1)))] + + after_id = None + for id_list in record_ids_paginated: + for property_slice in parsed_properties: + record_responses = [ + { + "json": { + "results": [ + {**self.BASE_OBJECT_BODY, **{"id": id, "properties": {p: "fake_data" for p in property_slice}}} + for id in id_list + ], + "paging": {"next": {"after": id_list[-1]}} if len(id_list) == 100 else {}, + }, + "status_code": 200, + } + ] + requests_mock.register_uri( + "GET", + f"{test_stream.url}?limit=100&properties={','.join(property_slice)}{f'&after={after_id}' if after_id else ''}", + record_responses, + ) + after_id = id_list[-1] + + # Read preudo-output from generator object read(), based on real scenario + stream_records = list(test_stream.read(getter=partial(self.get, test_stream.url, api=api))) + + # check that we have records for all set ids, and that each record has 2000 properties (not more, and not less) + assert len(stream_records) == sum([len(ids) for ids in record_ids_paginated]) + for record in stream_records: + assert len(record["properties"]) == NUMBER_OF_PROPERTIES + + def test_stream_with_splitting_properties_with_pagination(self, requests_mock, client, api, fake_properties_list): + """ + Check working stream `products` with large list of properties using new functionality with splitting properties + """ + stream_name = "products" + + parsed_properties = list(split_properties(fake_properties_list)) + self.set_mock_properties(requests_mock, "/properties/v2/product/properties", fake_properties_list) + test_stream = client._apis.get(stream_name) + + for property_slice in parsed_properties: + record_responses = [ + { + "json": { + "results": [ + {**self.BASE_OBJECT_BODY, **{"id": id, "properties": {p: "fake_data" for p in property_slice}}} + for id in ["6043593519", "1092593519", "1092593518", "1092593517", "1092593516"] + ], + "paging": {}, + }, + "status_code": 200, + } + ] + requests_mock.register_uri("GET", f"{test_stream.url}?properties={','.join(property_slice)}", record_responses) + + stream_records = list(test_stream.read(getter=partial(self.get, test_stream.url, api=api))) + + assert len(stream_records) == 5 + for record in stream_records: + assert len(record["properties"]) == NUMBER_OF_PROPERTIES + + def test_stream_with_splitting_properties_with_new_record(self, requests_mock, client, api, fake_properties_list): + """ + Check working stream `workflows` with large list of properties using new functionality with splitting properties + """ + stream_name = "deals" + + parsed_properties = list(split_properties(fake_properties_list)) + self.set_mock_properties(requests_mock, "/properties/v2/deal/properties", fake_properties_list) + + # Create test_stream instance + test_stream = client._apis.get(stream_name) + + ids_list = ["6043593519", "1092593519", "1092593518", "1092593517", "1092593516"] + for property_slice in parsed_properties: + record_responses = [ + { + "json": { + "results": [ + {**self.BASE_OBJECT_BODY, **{"id": id, "properties": {p: "fake_data" for p in property_slice}}} + for id in ids_list + ], + "paging": {}, + }, + "status_code": 200, + } + ] + requests_mock.register_uri("GET", f"{test_stream.url}?properties={','.join(property_slice)}", record_responses) + ids_list.append("1092593513") + + stream_records = list(test_stream.read(getter=partial(self.get, test_stream.url, api=api))) + + assert len(stream_records) == 6 From 736eafe5fe4728140043fe31bd073a77749e21f0 Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Mon, 25 Oct 2021 17:52:16 +0300 Subject: [PATCH 10/11] uncomment tests --- .../source-hubspot/unit_tests/test_client.py | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py index e67e27f221ae1..d40b0f292d142 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py @@ -27,35 +27,35 @@ def fake_properties_list(): return [f"property_number_{i}" for i in range(NUMBER_OF_PROPERTIES)] -# def test_client_backoff_on_limit_reached(requests_mock, some_credentials): -# """Error once, check that we retry and not fail""" -# responses = [ -# {"json": {"error": "limit reached"}, "status_code": 429, "headers": {"Retry-After": "0"}}, -# {"json": [], "status_code": 200}, -# ] -# -# requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) -# client = Client(start_date="2021-02-01T00:00:00Z", credentials=some_credentials) -# -# alive, error = client.health_check() -# -# assert alive -# assert not error - - -# def test_client_backoff_on_server_error(requests_mock, some_credentials): -# """Error once, check that we retry and not fail""" -# responses = [ -# {"json": {"error": "something bad"}, "status_code": 500}, -# {"json": [], "status_code": 200}, -# ] -# requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) -# client = Client(start_date="2021-02-01T00:00:00Z", credentials=some_credentials) -# -# alive, error = client.health_check() -# -# assert alive -# assert not error +def test_client_backoff_on_limit_reached(requests_mock, some_credentials): + """Error once, check that we retry and not fail""" + responses = [ + {"json": {"error": "limit reached"}, "status_code": 429, "headers": {"Retry-After": "0"}}, + {"json": [], "status_code": 200}, + ] + + requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) + client = Client(start_date="2021-02-01T00:00:00Z", credentials=some_credentials) + + alive, error = client.health_check() + + assert alive + assert not error + + +def test_client_backoff_on_server_error(requests_mock, some_credentials): + """Error once, check that we retry and not fail""" + responses = [ + {"json": {"error": "something bad"}, "status_code": 500}, + {"json": [], "status_code": 200}, + ] + requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) + client = Client(start_date="2021-02-01T00:00:00Z", credentials=some_credentials) + + alive, error = client.health_check() + + assert alive + assert not error def test_wrong_permissions_api_key(requests_mock, creds_with_wrong_permissions): From 10f580cfec7b0cfc74c69b70f1dda3c1db64486f Mon Sep 17 00:00:00 2001 From: ykurochkin Date: Tue, 26 Oct 2021 15:17:15 +0300 Subject: [PATCH 11/11] bump version --- .../36c891d9-4bd9-43ac-bad2-10e12756272c.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- docs/integrations/sources/hubspot.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json index 9b8b8dbbce1fd..3c05f6cb9bf4b 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "36c891d9-4bd9-43ac-bad2-10e12756272c", "name": "Hubspot", "dockerRepository": "airbyte/source-hubspot", - "dockerImageTag": "0.1.18", + "dockerImageTag": "0.1.19", "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot", "icon": "hubspot.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0b8509c994ef6..e8b509523efbe 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -161,7 +161,7 @@ - sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c name: Hubspot dockerRepository: airbyte/source-hubspot - dockerImageTag: 0.1.18 + dockerImageTag: 0.1.19 documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot icon: hubspot.svg sourceType: api diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 1e3d9d8a7c76f..1f8b7a7afbddc 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -96,7 +96,7 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.19 | 2021-10-?? | [6954](https://github.com/airbytehq/airbyte/pull/6954) | Fix issue with getting `414` HTTP error for streams | +| 0.1.19 | 2021-10-26 | [6954](https://github.com/airbytehq/airbyte/pull/6954) | Fix issue with getting `414` HTTP error for streams | | 0.1.18 | 2021-10-18 | [5840](https://github.com/airbytehq/airbyte/pull/5840) | Add new marketing emails (with statistics) stream | | 0.1.17 | 2021-10-14 | [6995](https://github.com/airbytehq/airbyte/pull/6995) | Update `discover` method: disable `quotes` stream when using OAuth config | | 0.1.16 | 2021-09-27 | [6465](https://github.com/airbytehq/airbyte/pull/6465) | Implement OAuth support. Use CDK authenticator instead of connector specific authenticator |