From 0f881644ba9e1fd5d1f6280fe23766698a2d295b Mon Sep 17 00:00:00 2001 From: Dom DiPasquale Date: Fri, 13 Dec 2024 13:50:31 -0500 Subject: [PATCH 1/6] bump cdk of select connectors to 1.8.0 --- .../connectors/source-appsflyer/pyproject.toml | 6 +++--- .../connectors/source-braintree/pyproject.toml | 2 +- .../connectors/source-commercetools/pyproject.toml | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-appsflyer/pyproject.toml b/airbyte-integrations/connectors/source-appsflyer/pyproject.toml index 41ff195982a4f..cc2ea31145392 100644 --- a/airbyte-integrations/connectors/source-appsflyer/pyproject.toml +++ b/airbyte-integrations/connectors/source-appsflyer/pyproject.toml @@ -1,12 +1,12 @@ [build-system] -requires = [ "poetry-core>=1.0.0",] +requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry] version = "0.2.22" name = "source-appsflyer" description = "Source implementation for Appsflyer." -authors = [ "Airbyte ",] +authors = ["Airbyte "] license = "MIT" readme = "README.md" documentation = "https://docs.airbyte.com/integrations/sources/appsflyer" @@ -17,7 +17,7 @@ include = "source_appsflyer" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "0.80.0" +airbyte-cdk = "1.8.0" pendulum = "==2.1.2" [tool.poetry.scripts] diff --git a/airbyte-integrations/connectors/source-braintree/pyproject.toml b/airbyte-integrations/connectors/source-braintree/pyproject.toml index 9bf04ade44c8d..f5a488116c784 100644 --- a/airbyte-integrations/connectors/source-braintree/pyproject.toml +++ b/airbyte-integrations/connectors/source-braintree/pyproject.toml @@ -17,7 +17,7 @@ include = "source_braintree" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "^0.50" +airbyte-cdk = "1.8.0" braintree = "^4.28.0" [tool.poetry.scripts] diff --git a/airbyte-integrations/connectors/source-commercetools/pyproject.toml b/airbyte-integrations/connectors/source-commercetools/pyproject.toml index 21ca0e31e402b..c89abbc07c920 100644 --- a/airbyte-integrations/connectors/source-commercetools/pyproject.toml +++ b/airbyte-integrations/connectors/source-commercetools/pyproject.toml @@ -1,12 +1,12 @@ [build-system] -requires = [ "poetry-core>=1.0.0",] +requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry] version = "0.2.24" name = "source-commercetools" description = "Source implementation for Commercetools." -authors = [ "Airbyte ",] +authors = ["Airbyte "] license = "MIT" readme = "README.md" documentation = "https://docs.airbyte.com/integrations/sources/commercetools" @@ -17,7 +17,7 @@ include = "source_commercetools" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "1.0.0" +airbyte-cdk = "1.8.0" [tool.poetry.scripts] source-commercetools = "source_commercetools.run:run" From 55b8859f9462e51c8210c7e1b3bc69b9019fc47f Mon Sep 17 00:00:00 2001 From: Dom DiPasquale Date: Fri, 13 Dec 2024 16:00:40 -0500 Subject: [PATCH 2/6] fix commercetools auth --- .../source_commercetools/components.py | 5 ++++- .../source_commercetools/manifest.yaml | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py b/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py index f39ab7dd6f286..934b7be4d4759 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py @@ -27,7 +27,10 @@ def _get_refresh_access_token_response(self): region = self.config["region"] project_key = self.config["project_key"] host = self.config["host"] - url = f"https://auth.{region}.{host}.commercetools.com/oauth/token?grant_type=client_credentials&scope=manage_project:{project_key}" + logger.info(f"Refreshing access token for project {project_key}") + scopes = map(lambda s: s + ":" + project_key, self.config["scopes"].split(",")) + scope_request = " ".join(scopes) + url = f"https://auth.{region}.{host}.commercetools.com/oauth/token?grant_type=client_credentials&scope={scope_request}" try: response = requests.post(url, auth=(self.config["client_id"], self.config["client_secret"])) response.raise_for_status() diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml b/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml index 8bc1b738f224b..9361f12dccd85 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml @@ -43,8 +43,6 @@ definitions: class_name: source_commercetools.components.CommerceToolsOauth2Authenticator client_id: "{{ config['client_id'] }}" client_secret: "{{ config['client_secret'] }}" - refresh_request_body: - scope: manage_project:{{ config['project_key'] }} token_refresh_endpoint: https://auth.{{ config['region'] }}.{{ config['host'] }}.commercetools.com/oauth/token grant_type: client_credentials access_token_name: access_token @@ -176,3 +174,8 @@ spec: type: string examples: ["us-central1", "australia-southeast1"] order: 5 + scopes: + title: Authenticated Scopes + description: "The comma-separated scopes required to authenticate with the API." + type: string + order: 6 From fca3ebd08b183eed99503d5770dd15377b8d6c90 Mon Sep 17 00:00:00 2001 From: Matthew Winslow Date: Thu, 23 Jan 2025 22:46:42 -0800 Subject: [PATCH 3/6] PRODUCT-4129 PRODUCT-4130 - implementing carts and inventory streams via manifest.yaml --- .../integration_tests/configured_catalog.json | 194 +- .../source_commercetools/manifest.yaml | 14 + .../source_commercetools/schemas/carts.json | 1589 +++++++++++++++++ .../schemas/inventory.json | 100 ++ 4 files changed, 1868 insertions(+), 29 deletions(-) create mode 100644 airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/carts.json create mode 100644 airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/inventory.json diff --git a/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json index 819e9f44cf81a..9739ec9740be5 100644 --- a/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json @@ -1,64 +1,200 @@ { "streams": [ { + "cursor_field": null, + "destination_sync_mode": "append", + "generation_id": null, + "minimum_generation_id": null, + "primary_key": null, "stream": { + "default_cursor_field": [ + "lastModifiedAt" + ], + "is_resumable": true, + "json_schema": {}, "name": "customers", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [ + [ + "id" + ] + ], + "supported_sync_modes": [ + "full_refresh", + "incremental" + ] + }, + "sync_id": null, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "generation_id": null, + "minimum_generation_id": null, + "primary_key": null, + "stream": { + "default_cursor_field": [ + "lastModifiedAt" + ], + "is_resumable": true, "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"], + "name": "discount_codes", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["lastModifiedAt"] + "source_defined_primary_key": [ + [ + "id" + ] + ], + "supported_sync_modes": [ + "full_refresh", + "incremental" + ] }, - "sync_mode": "incremental", - "cursor_field": ["lastModifiedAt"], - "destination_sync_mode": "append" + "sync_id": null, + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "generation_id": null, + "minimum_generation_id": null, + "primary_key": null, "stream": { + "default_cursor_field": [ + "lastModifiedAt" + ], + "is_resumable": true, + "json_schema": {}, "name": "orders", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [ + [ + "id" + ] + ], + "supported_sync_modes": [ + "full_refresh", + "incremental" + ] + }, + "sync_id": null, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "generation_id": null, + "minimum_generation_id": null, + "primary_key": null, + "stream": { + "default_cursor_field": [ + "lastModifiedAt" + ], + "is_resumable": true, "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"], + "name": "payments", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["lastModifiedAt"] + "source_defined_primary_key": [ + [ + "id" + ] + ], + "supported_sync_modes": [ + "full_refresh", + "incremental" + ] }, - "sync_mode": "incremental", - "cursor_field": ["lastModifiedAt"], - "destination_sync_mode": "append" + "sync_id": null, + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "generation_id": null, + "minimum_generation_id": null, + "primary_key": null, "stream": { - "name": "discount_codes", + "default_cursor_field": [ + "lastModifiedAt" + ], + "is_resumable": true, "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"], + "name": "products", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["lastModifiedAt"] + "source_defined_primary_key": [ + [ + "id" + ] + ], + "supported_sync_modes": [ + "full_refresh", + "incremental" + ] }, - "sync_mode": "incremental", - "cursor_field": ["lastModifiedAt"], - "destination_sync_mode": "append" + "sync_id": null, + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "generation_id": null, + "minimum_generation_id": null, + "primary_key": null, "stream": { - "name": "payments", + "default_cursor_field": [ + "lastModifiedAt" + ], + "is_resumable": true, "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"], + "name": "carts", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["lastModifiedAt"] + "source_defined_primary_key": [ + [ + "id" + ] + ], + "supported_sync_modes": [ + "full_refresh", + "incremental" + ] }, - "sync_mode": "incremental", - "cursor_field": ["lastModifiedAt"], - "destination_sync_mode": "append" + "sync_id": null, + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "generation_id": null, + "minimum_generation_id": null, + "primary_key": null, "stream": { - "name": "products", + "default_cursor_field": [ + "lastModifiedAt" + ], + "is_resumable": true, "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"], + "name": "inventory", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["lastModifiedAt"] + "source_defined_primary_key": [ + [ + "id" + ] + ], + "supported_sync_modes": [ + "full_refresh", + "incremental" + ] }, - "sync_mode": "incremental", - "cursor_field": ["lastModifiedAt"], - "destination_sync_mode": "append" + "sync_id": null, + "sync_mode": "full_refresh" } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml b/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml index 9361f12dccd85..b055c6166881a 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml @@ -114,12 +114,26 @@ definitions: name: "products" path: "products" + carts_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "carts" + path: "carts" + + inventory_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "inventory" + path: "inventory" + streams: - "#/definitions/customers_stream" - "#/definitions/discount_codes_stream" - "#/definitions/orders_stream" - "#/definitions/payments_stream" - "#/definitions/products_stream" + - "#/definitions/carts_stream" + - "#/definitions/inventory_stream" spec: documentation_url: https://docs.airbyte.com/integrations/sources/commercetools diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/carts.json b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/carts.json new file mode 100644 index 0000000000000..8856b9108ca1d --- /dev/null +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/carts.json @@ -0,0 +1,1589 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "billingAddress": { + "additionalProperties": true, + "properties": { + "additionalAddressInfo": { + "type": "string" + }, + "additionalStreetInfo": { + "type": "string" + }, + "city": { + "type": "string" + }, + "country": { + "type": "string" + }, + "firstName": { + "type": "string" + }, + "id": { + "type": "string" + }, + "lastName": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "postalCode": { + "type": "string" + }, + "region": { + "type": "string" + }, + "state": { + "type": "string" + }, + "streetName": { + "type": "string" + } + }, + "type": "object" + }, + "cartState": { + "type": "string" + }, + "createdAt": { + "type": "string" + }, + "createdBy": { + "additionalProperties": true, + "properties": { + "clientId": { + "type": "string" + }, + "isPlatformClient": { + "type": "boolean" + } + }, + "type": "object" + }, + "custom": { + "additionalProperties": true, + "properties": { + "fields": { + "additionalProperties": true, + "properties": { + "isAgeOfMajority": { + "type": "boolean" + } + }, + "type": "object" + }, + "type": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "customLineItems": { + "items": { + "additionalProperties": true, + "properties": { + "discountedPricePerQuantity": { + "type": "array" + }, + "id": { + "type": "string" + }, + "key": { + "type": "string" + }, + "money": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "name": { + "additionalProperties": true, + "properties": { + "en": { + "type": "string" + }, + "es": { + "type": "string" + }, + "fr": { + "type": "string" + } + }, + "type": "object" + }, + "perMethodTaxRate": { + "type": "array" + }, + "priceMode": { + "type": "string" + }, + "quantity": { + "type": "integer" + }, + "slug": { + "type": "string" + }, + "state": { + "items": { + "additionalProperties": true, + "properties": { + "quantity": { + "type": "integer" + }, + "state": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "type": "array" + }, + "taxCategory": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "taxRate": { + "additionalProperties": true, + "properties": { + "amount": { + "type": "number" + }, + "country": { + "type": "string" + }, + "includedInPrice": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "subRates": { + "type": "array" + } + }, + "type": "object" + }, + "taxedPrice": { + "additionalProperties": true, + "properties": { + "taxPortions": { + "items": { + "additionalProperties": true, + "properties": { + "amount": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "rate": { + "type": "number" + } + }, + "type": "object" + }, + "type": "array" + }, + "totalGross": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "totalNet": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "totalTax": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "taxedPricePortions": { + "type": "array" + }, + "totalPrice": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "type": "array" + }, + "customerEmail": { + "type": "string" + }, + "customerGroup": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "customerId": { + "type": "string" + }, + "deleteDaysAfterLastModification": { + "type": "integer" + }, + "directDiscounts": { + "type": "array" + }, + "discountCodes": { + "items": { + "additionalProperties": true, + "properties": { + "discountCode": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "state": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "inventoryMode": { + "type": "string" + }, + "itemShippingAddresses": { + "type": "array" + }, + "lastMessageSequenceNumber": { + "type": "integer" + }, + "lastModifiedAt": { + "type": "string" + }, + "lastModifiedBy": { + "additionalProperties": true, + "properties": { + "clientId": { + "type": "string" + }, + "isPlatformClient": { + "type": "boolean" + } + }, + "type": "object" + }, + "lineItems": { + "items": { + "additionalProperties": true, + "properties": { + "addedAt": { + "type": "string" + }, + "custom": { + "additionalProperties": true, + "properties": { + "fields": { + "additionalProperties": true, + "properties": { + "external-id": { + "type": "string" + }, + "parent": { + "type": "string" + } + }, + "type": "object" + }, + "type": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "discountedPrice": { + "additionalProperties": true, + "properties": { + "includedDiscounts": { + "items": { + "additionalProperties": true, + "properties": { + "discount": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "discountedAmount": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "type": "array" + }, + "value": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "discountedPricePerQuantity": { + "items": { + "additionalProperties": true, + "properties": { + "discountedPrice": { + "additionalProperties": true, + "properties": { + "includedDiscounts": { + "items": { + "additionalProperties": true, + "properties": { + "discount": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "discountedAmount": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "type": "array" + }, + "value": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "quantity": { + "type": "integer" + } + }, + "type": "object" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "lastModifiedAt": { + "type": "string" + }, + "lineItemMode": { + "type": "string" + }, + "name": { + "additionalProperties": true, + "properties": { + "en-US": { + "type": "string" + }, + "es-MX": { + "type": "string" + } + }, + "type": "object" + }, + "perMethodTaxRate": { + "type": "array" + }, + "price": { + "additionalProperties": true, + "properties": { + "customerGroup": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "id": { + "type": "string" + }, + "validFrom": { + "type": "string" + }, + "validUntil": { + "type": "string" + }, + "value": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "priceMode": { + "type": "string" + }, + "productId": { + "type": "string" + }, + "productKey": { + "type": "string" + }, + "productSlug": { + "additionalProperties": true, + "properties": { + "en-US": { + "type": "string" + }, + "es-MX": { + "type": "string" + } + }, + "type": "object" + }, + "productType": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + }, + "version": { + "type": "integer" + } + }, + "type": "object" + }, + "quantity": { + "type": "integer" + }, + "state": { + "items": { + "additionalProperties": true, + "properties": { + "quantity": { + "type": "integer" + }, + "state": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "type": "array" + }, + "supplyChannel": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "taxRate": { + "additionalProperties": true, + "properties": { + "amount": { + "type": "number" + }, + "country": { + "type": "string" + }, + "includedInPrice": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "subRates": { + "type": "array" + } + }, + "type": "object" + }, + "taxedPrice": { + "additionalProperties": true, + "properties": { + "taxPortions": { + "items": { + "additionalProperties": true, + "properties": { + "amount": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "rate": { + "type": "number" + } + }, + "type": "object" + }, + "type": "array" + }, + "totalGross": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "totalNet": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "totalTax": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "taxedPricePortions": { + "type": "array" + }, + "totalPrice": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "variant": { + "additionalProperties": true, + "properties": { + "assets": { + "type": "array" + }, + "attributes": { + "items": { + "additionalProperties": true, + "properties": { + "name": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": [ + "boolean", + "number", + "string" + ] + }, + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "properties": { + "en-US": { + "type": "string" + }, + "es-MX": { + "type": "string" + }, + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + { + "items": { + "additionalProperties": true, + "properties": { + "name": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "integer" + }, + { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + } + ] + } + }, + "type": "object" + }, + "type": "array" + } + ] + }, + "type": "array" + }, + { + "additionalProperties": true, + "properties": { + "en-US": { + "type": "string" + }, + "es-MX": { + "type": "string" + }, + "id": { + "type": "string" + }, + "key": { + "type": "string" + }, + "label": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + } + ] + } + }, + "type": "object" + }, + "type": "array" + }, + "availability": { + "additionalProperties": true, + "properties": { + "availableQuantity": { + "type": "integer" + }, + "channels": { + "additionalProperties": true, + "properties": { + "1fd6c4cf-ac1b-4bf2-a3f5-8a6f788719e7": { + "additionalProperties": true, + "properties": { + "availableQuantity": { + "type": "integer" + }, + "id": { + "type": "string" + }, + "isOnStock": { + "type": "boolean" + }, + "version": { + "type": "integer" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "id": { + "type": "string" + }, + "isOnStock": { + "type": "boolean" + }, + "version": { + "type": "integer" + } + }, + "type": "object" + }, + "id": { + "type": "integer" + }, + "images": { + "items": { + "additionalProperties": true, + "properties": { + "dimensions": { + "additionalProperties": true, + "properties": { + "h": { + "type": "integer" + }, + "w": { + "type": "integer" + } + }, + "type": "object" + }, + "label": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "key": { + "type": "string" + }, + "prices": { + "type": "array" + }, + "sku": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "type": "array" + }, + "locale": { + "type": "string" + }, + "origin": { + "type": "string" + }, + "paymentInfo": { + "additionalProperties": true, + "properties": { + "payments": { + "items": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "refusedGifts": { + "items": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "shipping": { + "type": "array" + }, + "shippingAddress": { + "additionalProperties": true, + "properties": { + "additionalAddressInfo": { + "type": "string" + }, + "additionalStreetInfo": { + "type": "string" + }, + "city": { + "type": "string" + }, + "country": { + "type": "string" + }, + "firstName": { + "type": "string" + }, + "id": { + "type": "string" + }, + "lastName": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "postalCode": { + "type": "string" + }, + "region": { + "type": "string" + }, + "state": { + "type": "string" + }, + "streetName": { + "type": "string" + } + }, + "type": "object" + }, + "shippingInfo": { + "additionalProperties": true, + "properties": { + "deliveries": { + "type": "array" + }, + "price": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "shippingMethod": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "shippingMethodName": { + "type": "string" + }, + "shippingMethodState": { + "type": "string" + }, + "shippingRate": { + "additionalProperties": true, + "properties": { + "freeAbove": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "price": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "tiers": { + "type": "array" + } + }, + "type": "object" + }, + "taxRate": { + "additionalProperties": true, + "properties": { + "amount": { + "type": "number" + }, + "country": { + "type": "string" + }, + "includedInPrice": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "subRates": { + "type": "array" + } + }, + "type": "object" + }, + "taxedPrice": { + "additionalProperties": true, + "properties": { + "taxPortions": { + "items": { + "additionalProperties": true, + "properties": { + "amount": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "rate": { + "type": "number" + } + }, + "type": "object" + }, + "type": "array" + }, + "totalGross": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "totalNet": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "totalTax": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "shippingMode": { + "type": "string" + }, + "store": { + "additionalProperties": true, + "properties": { + "key": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "taxCalculationMode": { + "type": "string" + }, + "taxMode": { + "type": "string" + }, + "taxRoundingMode": { + "type": "string" + }, + "taxedPrice": { + "additionalProperties": true, + "properties": { + "taxPortions": { + "items": { + "additionalProperties": true, + "properties": { + "amount": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "rate": { + "type": "number" + } + }, + "type": "object" + }, + "type": "array" + }, + "totalGross": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "totalNet": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "totalTax": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "taxedShippingPrice": { + "additionalProperties": true, + "properties": { + "taxPortions": { + "items": { + "additionalProperties": true, + "properties": { + "amount": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "rate": { + "type": "number" + } + }, + "type": "object" + }, + "type": "array" + }, + "totalGross": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "totalNet": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "totalTax": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "totalLineItemQuantity": { + "type": "integer" + }, + "totalPrice": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "type": { + "type": "string" + }, + "version": { + "type": "integer" + }, + "versionModifiedAt": { + "type": "string" + } + }, + "type": "object" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/inventory.json b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/inventory.json new file mode 100644 index 0000000000000..a5f6076039d9e --- /dev/null +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/inventory.json @@ -0,0 +1,100 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "availableQuantity": { + "type": "integer" + }, + "createdAt": { + "type": "string" + }, + "createdBy": { + "additionalProperties": true, + "properties": { + "clientId": { + "type": "string" + }, + "isPlatformClient": { + "type": "boolean" + }, + "user": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "id": { + "type": "string" + }, + "key": { + "type": "string" + }, + "lastMessageSequenceNumber": { + "type": "integer" + }, + "lastModifiedAt": { + "type": "string" + }, + "lastModifiedBy": { + "additionalProperties": true, + "properties": { + "clientId": { + "type": "string" + }, + "isPlatformClient": { + "type": "boolean" + }, + "user": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "quantityOnStock": { + "type": "integer" + }, + "reservations": { + "type": "array" + }, + "sku": { + "type": "string" + }, + "supplyChannel": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "version": { + "type": "integer" + }, + "versionModifiedAt": { + "type": "string" + } + }, + "type": "object" +} \ No newline at end of file From 1c3c21c01066207e706c3b88128574a3805ffc31 Mon Sep 17 00:00:00 2001 From: Matthew Winslow Date: Wed, 29 Jan 2025 13:56:02 -0800 Subject: [PATCH 4/6] PRODUCT-4202 - adding standalone-prices endpoint. --- .../integration_tests/configured_catalog.json | 28 +++ .../source_commercetools/manifest.yaml | 7 + .../schemas/standalone-prices.json | 214 ++++++++++++++++++ 3 files changed, 249 insertions(+) create mode 100644 airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/standalone-prices.json diff --git a/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json index 9739ec9740be5..47ad8c73e25f5 100644 --- a/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json @@ -195,6 +195,34 @@ }, "sync_id": null, "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "generation_id": null, + "minimum_generation_id": null, + "primary_key": null, + "stream": { + "default_cursor_field": [ + "lastModifiedAt" + ], + "is_resumable": true, + "json_schema": {}, + "name": "standalone-prices", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [ + [ + "id" + ] + ], + "supported_sync_modes": [ + "full_refresh", + "incremental" + ] + }, + "sync_id": null, + "sync_mode": "full_refresh" } ] } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml b/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml index b055c6166881a..6e67287164a19 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml @@ -126,6 +126,12 @@ definitions: name: "inventory" path: "inventory" + standalone-prices_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "standalone-prices" + path: "standalone-prices" + streams: - "#/definitions/customers_stream" - "#/definitions/discount_codes_stream" @@ -134,6 +140,7 @@ streams: - "#/definitions/products_stream" - "#/definitions/carts_stream" - "#/definitions/inventory_stream" + - "#/definitions/standalone-prices_stream" spec: documentation_url: https://docs.airbyte.com/integrations/sources/commercetools diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/standalone-prices.json b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/standalone-prices.json new file mode 100644 index 0000000000000..5173c194fd0b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/standalone-prices.json @@ -0,0 +1,214 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "active": { + "type": "boolean" + }, + "createdAt": { + "type": "string" + }, + "createdBy": { + "additionalProperties": true, + "properties": { + "clientId": { + "type": "string" + }, + "isPlatformClient": { + "type": "boolean" + }, + "user": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "customerGroup": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "discounted": { + "additionalProperties": true, + "properties": { + "discount": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "value": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "expiresAt": { + "type": "string" + }, + "id": { + "type": "string" + }, + "lastMessageSequenceNumber": { + "type": "integer" + }, + "lastModifiedAt": { + "type": "string" + }, + "lastModifiedBy": { + "additionalProperties": true, + "properties": { + "clientId": { + "type": "string" + }, + "isPlatformClient": { + "type": "boolean" + }, + "user": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "sku": { + "type": "string" + }, + "staged": { + "additionalProperties": true, + "properties": { + "discounted": { + "additionalProperties": true, + "properties": { + "discount": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "value": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "value": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "validFrom": { + "type": "string" + }, + "validUntil": { + "type": "string" + }, + "value": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "version": { + "type": "integer" + }, + "versionModifiedAt": { + "type": "string" + } + }, + "type": "object" +} \ No newline at end of file From 8c50b9840d43293dca9cf33497c8e8c9ee6dd1db Mon Sep 17 00:00:00 2001 From: Matthew Winslow Date: Fri, 31 Jan 2025 08:08:00 -0800 Subject: [PATCH 5/6] PRODUCT-4217 - adding product-discounts stream. - also updated carts schema. --- .../integration_tests/configured_catalog.json | 28 ++++ .../source_commercetools/manifest.yaml | 8 + .../source_commercetools/schemas/carts.json | 3 + .../schemas/product-discounts.json | 154 ++++++++++++++++++ 4 files changed, 193 insertions(+) create mode 100644 airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/product-discounts.json diff --git a/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json index 47ad8c73e25f5..0f93ffc16bfaa 100644 --- a/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json @@ -223,6 +223,34 @@ }, "sync_id": null, "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "generation_id": null, + "minimum_generation_id": null, + "primary_key": null, + "stream": { + "default_cursor_field": [ + "lastModifiedAt" + ], + "is_resumable": true, + "json_schema": {}, + "name": "product-discounts", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [ + [ + "id" + ] + ], + "supported_sync_modes": [ + "full_refresh", + "incremental" + ] + }, + "sync_id": null, + "sync_mode": "full_refresh" } ] } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml b/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml index 6e67287164a19..2165948317fdd 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml @@ -132,6 +132,13 @@ definitions: name: "standalone-prices" path: "standalone-prices" + product-discounts_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "product-discounts" + path: "product-discounts" + + streams: - "#/definitions/customers_stream" - "#/definitions/discount_codes_stream" @@ -141,6 +148,7 @@ streams: - "#/definitions/carts_stream" - "#/definitions/inventory_stream" - "#/definitions/standalone-prices_stream" + - "#/definitions/product-discounts_stream" spec: documentation_url: https://docs.airbyte.com/integrations/sources/commercetools diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/carts.json b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/carts.json index 8856b9108ca1d..26b1cf9e3700a 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/carts.json +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/carts.json @@ -576,6 +576,9 @@ "id": { "type": "string" }, + "inventoryMode": { + "type": "string" + }, "lastModifiedAt": { "type": "string" }, diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/product-discounts.json b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/product-discounts.json new file mode 100644 index 0000000000000..0f7f93ae5ccc1 --- /dev/null +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/product-discounts.json @@ -0,0 +1,154 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "createdAt": { + "type": "string" + }, + "createdBy": { + "additionalProperties": true, + "properties": { + "isPlatformClient": { + "type": "boolean" + }, + "user": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "description": { + "additionalProperties": true, + "properties": { + "en-CA": { + "type": "string" + }, + "en-US": { + "type": "string" + }, + "es-MX": { + "type": "string" + }, + "fr-CA": { + "type": "string" + } + }, + "type": "object" + }, + "id": { + "type": "string" + }, + "isActive": { + "type": "boolean" + }, + "key": { + "type": "string" + }, + "lastModifiedAt": { + "type": "string" + }, + "lastModifiedBy": { + "additionalProperties": true, + "properties": { + "isPlatformClient": { + "type": "boolean" + }, + "user": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "name": { + "additionalProperties": true, + "properties": { + "en-CA": { + "type": "string" + }, + "en-US": { + "type": "string" + }, + "es-MX": { + "type": "string" + }, + "fr-CA": { + "type": "string" + } + }, + "type": "object" + }, + "predicate": { + "type": "string" + }, + "references": { + "type": "array" + }, + "sortOrder": { + "type": "string" + }, + "validFrom": { + "type": "string" + }, + "validUntil": { + "type": "string" + }, + "value": { + "additionalProperties": true, + "properties": { + "money": { + "items": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "permyriad": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "version": { + "type": "integer" + }, + "versionModifiedAt": { + "type": "string" + } + }, + "type": "object" +} \ No newline at end of file From 996c81aff9537024503dd62c83b79da888f8cac4 Mon Sep 17 00:00:00 2001 From: Matthew Winslow Date: Fri, 14 Feb 2025 11:24:43 -0800 Subject: [PATCH 6/6] ACC-7752 - adding cart-discounts stream - adding helper script for refreshing catalog and schemas with some basic checks. --- .../integration_tests/configured_catalog.json | 28 +++ .../source_commercetools/manifest.yaml | 7 + .../schemas/cart-discounts.json | 226 ++++++++++++++++++ tools/bin/reactor-refresh-schema-source.sh | 35 +++ 4 files changed, 296 insertions(+) create mode 100644 airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/cart-discounts.json create mode 100755 tools/bin/reactor-refresh-schema-source.sh diff --git a/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json index 0f93ffc16bfaa..cf83410ff1765 100644 --- a/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json @@ -251,6 +251,34 @@ }, "sync_id": null, "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "generation_id": null, + "minimum_generation_id": null, + "primary_key": null, + "stream": { + "default_cursor_field": [ + "lastModifiedAt" + ], + "is_resumable": true, + "json_schema": {}, + "name": "cart-discounts", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [ + [ + "id" + ] + ], + "supported_sync_modes": [ + "full_refresh", + "incremental" + ] + }, + "sync_id": null, + "sync_mode": "full_refresh" } ] } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml b/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml index 2165948317fdd..dfa30ec359433 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml @@ -138,6 +138,12 @@ definitions: name: "product-discounts" path: "product-discounts" + cart-discounts_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "cart-discounts" + path: "cart-discounts" + streams: - "#/definitions/customers_stream" @@ -149,6 +155,7 @@ streams: - "#/definitions/inventory_stream" - "#/definitions/standalone-prices_stream" - "#/definitions/product-discounts_stream" + - "#/definitions/cart-discounts_stream" spec: documentation_url: https://docs.airbyte.com/integrations/sources/commercetools diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/cart-discounts.json b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/cart-discounts.json new file mode 100644 index 0000000000000..5b0c8d0dc7077 --- /dev/null +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/cart-discounts.json @@ -0,0 +1,226 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "cartPredicate": { + "type": "string" + }, + "createdAt": { + "type": "string" + }, + "createdBy": { + "additionalProperties": true, + "properties": { + "clientId": { + "type": "string" + }, + "isPlatformClient": { + "type": "boolean" + }, + "user": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "description": { + "additionalProperties": true, + "properties": { + "en-CA": { + "type": "string" + }, + "en-US": { + "type": "string" + }, + "es-MX": { + "type": "string" + }, + "fr-CA": { + "type": "string" + } + }, + "type": "object" + }, + "id": { + "type": "string" + }, + "isActive": { + "type": "boolean" + }, + "key": { + "type": "string" + }, + "lastMessageSequenceNumber": { + "type": "integer" + }, + "lastModifiedAt": { + "type": "string" + }, + "lastModifiedBy": { + "additionalProperties": true, + "properties": { + "clientId": { + "type": "string" + }, + "isPlatformClient": { + "type": "boolean" + }, + "user": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "name": { + "additionalProperties": true, + "properties": { + "en-CA": { + "type": "string" + }, + "en-US": { + "type": "string" + }, + "es-MX": { + "type": "string" + }, + "fr-CA": { + "type": "string" + } + }, + "type": "object" + }, + "references": { + "type": "array" + }, + "requiresDiscountCode": { + "type": "boolean" + }, + "sortOrder": { + "type": "string" + }, + "stackingMode": { + "type": "string" + }, + "stores": { + "items": { + "additionalProperties": true, + "properties": { + "key": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "target": { + "additionalProperties": true, + "properties": { + "predicate": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "validFrom": { + "type": "string" + }, + "validUntil": { + "type": "string" + }, + "value": { + "additionalProperties": true, + "properties": { + "applicationMode": { + "type": "string" + }, + "money": { + "items": { + "additionalProperties": true, + "properties": { + "centAmount": { + "type": "integer" + }, + "currencyCode": { + "type": "string" + }, + "fractionDigits": { + "type": "integer" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "permyriad": { + "type": "integer" + }, + "product": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "supplyChannel": { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "typeId": { + "type": "string" + } + }, + "type": "object" + }, + "type": { + "type": "string" + }, + "variantId": { + "type": "integer" + } + }, + "type": "object" + }, + "version": { + "type": "integer" + }, + "versionModifiedAt": { + "type": "string" + } + }, + "type": "object" +} \ No newline at end of file diff --git a/tools/bin/reactor-refresh-schema-source.sh b/tools/bin/reactor-refresh-schema-source.sh new file mode 100755 index 0000000000000..29b19156fc879 --- /dev/null +++ b/tools/bin/reactor-refresh-schema-source.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +which schema_generator > /dev/null +if [ $? != 0 ];then + echo "schema_generator is not installed. See https://github.com/airbytehq/airbyte/tree/master/tools/schema_generator/ for installation instructions" +fi +root=$(git rev-parse --show-toplevel 2>> /dev/null) +source=$1 +airbyte_config=$2 +source_root="${root}/airbyte-integrations/connectors/source-${source}" +if [ ! --d "${source_root}" ];then + echo "Invalid source specified . Source folder not found at ${source_root}" + exit 1 +fi + +if [ -z "${airbyte_config}" ];then + echo "airbyte config must be provided" + exit 1 +fi +if [ ! -f "${airbyte_config}" ];then + echo "${airbyte_config} is not a valid path" + exit 1 +fi +pushd $PWD > /dev/null +cd $source_root +echo "refreshing catalog for source" +python main.py discover --config $airbyte_config | schema_generator --configure-catalog + + +echo "refreshing schemas for source. (note: This is fetching live data using the provided credentials. it may take a while." +cd source_commercetools +python ../main.py read --config $airbyte_config --catalog ../integration_tests/configured_catalog.json | schema_generator --infer-schemas +echo "Finished refreshing schemas. NOTE: Some schemas have been tweaked and standardized by hand. You probably only want to check in the new schema for the no stream that was added." +echo "$source_root/integration_tests/configured_catalog.json should be checked in along with any new schemas found at $source_root/source_${source}/schemas/" +popd > /dev/null \ No newline at end of file