From b6304ceca7d7a07b1fe8d59fe52f1972eff6c749 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Tue, 25 Jul 2023 11:04:27 -0300 Subject: [PATCH 1/8] make teradata avaialble in Airbyte Cloud --- .../connectors/destination-teradata/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-teradata/metadata.yaml b/airbyte-integrations/connectors/destination-teradata/metadata.yaml index 7185cb3293b89..75e3b27132d1f 100644 --- a/airbyte-integrations/connectors/destination-teradata/metadata.yaml +++ b/airbyte-integrations/connectors/destination-teradata/metadata.yaml @@ -10,7 +10,7 @@ data: name: Teradata Vantage registries: cloud: - enabled: false + enabled: true oss: enabled: true releaseStage: alpha From ec973380f43faeb747da3bad39b49cfb8194a1df Mon Sep 17 00:00:00 2001 From: SatishChGit Date: Mon, 18 Dec 2023 19:08:53 +0530 Subject: [PATCH 2/8] =?UTF-8?q?=E2=9C=A8Destination=20Teradata:=20Added=20?= =?UTF-8?q?ENCRYPTDATA=3DON=20for=20non-TLS=20connections=20for=20airbyte?= =?UTF-8?q?=20cloud=20(#32866)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../destination-teradata/Dockerfile | 2 +- .../destination-teradata/metadata.yaml | 2 +- .../teradata/TeradataDestination.java | 7 ++- docs/integrations/destinations/teradata.md | 50 ++++++++++--------- 4 files changed, 35 insertions(+), 26 deletions(-) diff --git a/airbyte-integrations/connectors/destination-teradata/Dockerfile b/airbyte-integrations/connectors/destination-teradata/Dockerfile index 655dea677eb07..96d74f9054edb 100644 --- a/airbyte-integrations/connectors/destination-teradata/Dockerfile +++ b/airbyte-integrations/connectors/destination-teradata/Dockerfile @@ -22,5 +22,5 @@ ENV APPLICATION destination-teradata COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-teradata diff --git a/airbyte-integrations/connectors/destination-teradata/metadata.yaml b/airbyte-integrations/connectors/destination-teradata/metadata.yaml index 75e3b27132d1f..e58f3fa5d8bd6 100644 --- a/airbyte-integrations/connectors/destination-teradata/metadata.yaml +++ b/airbyte-integrations/connectors/destination-teradata/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 58e6f9da-904e-11ed-a1eb-0242ac120002 - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.4 dockerRepository: airbyte/destination-teradata githubIssueLabel: destination-teradata icon: teradata.svg diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java index 61b0a2e708c64..c2717a17d86f8 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java +++ b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java @@ -50,6 +50,10 @@ public class TeradataDestination extends AbstractJdbcDestination implements Dest protected static final String CA_CERT_KEY = "ssl_ca_certificate"; + protected static final String ENCRYPTDATA = "ENCRYPTDATA"; + + protected static final String ENCRYPTDATA_ON = "ON"; + public static void main(String[] args) throws Exception { new IntegrationRunner(new TeradataDestination()).run(args); } @@ -69,7 +73,8 @@ protected Map getDefaultConnectionProperties(final JsonNode conf } else { additionalParameters.put(PARAM_SSLMODE, REQUIRE); } - } + } + additionalParameters.put(ENCRYPTDATA, ENCRYPTDATA_ON); return additionalParameters; } diff --git a/docs/integrations/destinations/teradata.md b/docs/integrations/destinations/teradata.md index 6ea038fce3ea0..ae06e826c1ac1 100644 --- a/docs/integrations/destinations/teradata.md +++ b/docs/integrations/destinations/teradata.md @@ -6,17 +6,17 @@ This page guides you through the process of setting up the Teradata destination To use the Teradata destination connector, you'll need: -* Access to a Teradata Vantage instance +- Access to a Teradata Vantage instance - **Note:** If you need a new instance of Vantage, you can install a free version called Vantage Express in the cloud on [Google Cloud](https://quickstarts.teradata.com/vantage.express.gcp.html), [Azure](https://quickstarts.teradata.com/run-vantage-express-on-microsoft-azure.html), and [AWS](https://quickstarts.teradata.com/run-vantage-express-on-aws.html). You can also run Vantage Express on your local machine using [VMware](https://quickstarts.teradata.com/getting.started.vmware.html), [VirtualBox](https://quickstarts.teradata.com/getting.started.vbox.html), or [UTM](https://quickstarts.teradata.com/getting.started.utm.html). + **Note:** If you need a new instance of Vantage, you can install a free version called Vantage Express in the cloud on [Google Cloud](https://quickstarts.teradata.com/vantage.express.gcp.html), [Azure](https://quickstarts.teradata.com/run-vantage-express-on-microsoft-azure.html), and [AWS](https://quickstarts.teradata.com/run-vantage-express-on-aws.html). You can also run Vantage Express on your local machine using [VMware](https://quickstarts.teradata.com/getting.started.vmware.html), [VirtualBox](https://quickstarts.teradata.com/getting.started.vbox.html), or [UTM](https://quickstarts.teradata.com/getting.started.utm.html). You'll need the following information to configure the Teradata destination: -* **Host** - The host name of the Teradata Vantage instance. -* **Username** -* **Password** -* **Default Schema Name** - Specify the schema (or several schemas separated by commas) to be set in the search-path. These schemas will be used to resolve unqualified object names used in statements executed over this connection. -* **JDBC URL Params** (optional) +- **Host** - The host name of the Teradata Vantage instance. +- **Username** +- **Password** +- **Default Schema Name** - Specify the schema (or several schemas separated by commas) to be set in the search-path. These schemas will be used to resolve unqualified object names used in statements executed over this connection. +- **JDBC URL Params** (optional) [Refer to this guide for more details](https://downloads.teradata.com/doc/connectivity/jdbc/reference/current/jdbcug_chapter_2.html#BGBHDDGB) @@ -26,23 +26,23 @@ You'll need the following information to configure the Teradata destination: Each stream will be output into its own table in Teradata. Each table will contain 3 columns: -* `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in Teradata is `VARCHAR(256)`. -* `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in Teradata is `TIMESTAMP(6)`. -* `_airbyte_data`: a json blob representing with the event data. The column type in Teradata is `JSON`. - +- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in Teradata is `VARCHAR(256)`. +- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in Teradata is `TIMESTAMP(6)`. +- `_airbyte_data`: a json blob representing with the event data. The column type in Teradata is `JSON`. ### Features The Teradata destination connector supports the following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): +| Feature | Supported?\(Yes/No\) | Notes | +| :----------------------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Incremental - Append + Deduped | No | | +| Namespaces | Yes | | -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental - Append Sync | Yes | | -| Incremental - Deduped History | No | | -| Namespaces | Yes | | +The Teradata destination connector supports [ DBT custom transformation](https://docs.airbyte.com/operator-guides/transformation-and-normalization/transformations-with-airbyte/) type. Teradata DBT Docker image is available at https://hub.docker.com/r/teradata/dbt-teradata. ### Performance considerations @@ -52,8 +52,8 @@ following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-s You need a Teradata user with the following permissions: -* can create tables and write permission. -* can create schemas e.g: +- can create tables and write permission. +- can create schemas e.g: You can create such a user by running: @@ -64,6 +64,7 @@ GRANT ALL on dbc to airbyte_user; ``` You can also use a pre-existing user but we highly recommend creating a dedicated user for Airbyte. + ### Setup guide #### Set up the Teradata Destination connector @@ -83,7 +84,10 @@ You can also use a pre-existing user but we highly recommend creating a dedicate ## CHANGELOG -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:--------------------------------------------------------------|:---------------------------------| -| 0.1.0 | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428 | New Destination Teradata Vantage | -| 0.1.1 | 2023-03-03 | https://github.com/airbytehq/airbyte/pull/21760 | Added SSL support | \ No newline at end of file +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :---------------------------------------------- | :---------------------------------------- | +| 0.1.4 | 2023-12-04 | https://github.com/airbytehq/airbyte/pull/28667 | Make connector available on Airbyte Cloud | +| 0.1.3 | 2023-08-17 | https://github.com/airbytehq/airbyte/pull/30740 | Enable custom DBT transformation | +| 0.1.2 | 2023-08-09 | https://github.com/airbytehq/airbyte/pull/29174 | Small internal refactor | +| 0.1.1 | 2023-03-03 | https://github.com/airbytehq/airbyte/pull/21760 | Added SSL support | +| 0.1.0 | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428 | New Destination Teradata Vantage | From ebbe005e03fbf4fb60d4ec1399fe9dd6ce4405e4 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Mon, 18 Dec 2023 11:05:39 -0300 Subject: [PATCH 3/8] fix conflict --- .../destination-teradata/metadata.yaml | 4 -- .../teradata/TeradataDestination.java | 57 ++++++------------- docs/integrations/destinations/teradata.md | 11 +--- 3 files changed, 19 insertions(+), 53 deletions(-) diff --git a/airbyte-integrations/connectors/destination-teradata/metadata.yaml b/airbyte-integrations/connectors/destination-teradata/metadata.yaml index 4ca9e1382641d..17e0de4e04939 100644 --- a/airbyte-integrations/connectors/destination-teradata/metadata.yaml +++ b/airbyte-integrations/connectors/destination-teradata/metadata.yaml @@ -2,11 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 58e6f9da-904e-11ed-a1eb-0242ac120002 -<<<<<<< HEAD dockerImageTag: 0.1.4 -======= - dockerImageTag: 0.1.3 ->>>>>>> master dockerRepository: airbyte/destination-teradata githubIssueLabel: destination-teradata icon: teradata.svg diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java index c8caa02062b3b..b8bf7ec01a197 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java +++ b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java @@ -49,14 +49,10 @@ public class TeradataDestination extends AbstractJdbcDestination implements Dest protected static final String CA_CERT_KEY = "ssl_ca_certificate"; -<<<<<<< HEAD - protected static final String ENCRYPTDATA = "ENCRYPTDATA"; + protected static final String ENCRYPTDATA = "ENCRYPTDATA"; - protected static final String ENCRYPTDATA_ON = "ON"; + protected static final String ENCRYPTDATA_ON = "ON"; - public static void main(String[] args) throws Exception { - new IntegrationRunner(new TeradataDestination()).run(args); -======= public static void main(String[] args) throws Exception { new IntegrationRunner(new TeradataDestination()).run(args); } @@ -65,44 +61,28 @@ public TeradataDestination() { super(DRIVER_CLASS, new StandardNameTransformer(), new TeradataSqlOperations()); } - @Override - protected Map getDefaultConnectionProperties(final JsonNode config) { - final Map additionalParameters = new HashMap<>(); - if (config.has(PARAM_SSL) && config.get(PARAM_SSL).asBoolean()) { - LOGGER.debug("SSL Enabled"); - if (config.has(PARAM_SSL_MODE)) { - LOGGER.debug("Selected SSL Mode : " + config.get(PARAM_SSL_MODE).get(PARAM_MODE).asText()); - additionalParameters.putAll(obtainConnectionOptions(config.get(PARAM_SSL_MODE))); - } else { - additionalParameters.put(PARAM_SSLMODE, REQUIRE); - } ->>>>>>> master - } - return additionalParameters; - } - private static void createCertificateFile(String fileName, String fileValue) throws IOException { try (final PrintWriter out = new PrintWriter(fileName, StandardCharsets.UTF_8)) { out.print(fileValue); } } -<<<<<<< HEAD - @Override - protected Map getDefaultConnectionProperties(final JsonNode config) { - final Map additionalParameters = new HashMap<>(); - if (config.has(PARAM_SSL) && config.get(PARAM_SSL).asBoolean()) { - LOGGER.debug("SSL Enabled"); - if (config.has(PARAM_SSL_MODE)) { - LOGGER.debug("Selected SSL Mode : " + config.get(PARAM_SSL_MODE).get(PARAM_MODE).asText()); - additionalParameters.putAll(obtainConnectionOptions(config.get(PARAM_SSL_MODE))); - } else { - additionalParameters.put(PARAM_SSLMODE, REQUIRE); - } - } - additionalParameters.put(ENCRYPTDATA, ENCRYPTDATA_ON); - return additionalParameters; -======= + @Override + protected Map getDefaultConnectionProperties(final JsonNode config) { + final Map additionalParameters = new HashMap<>(); + if (config.has(PARAM_SSL) && config.get(PARAM_SSL).asBoolean()) { + LOGGER.debug("SSL Enabled"); + if (config.has(PARAM_SSL_MODE)) { + LOGGER.debug("Selected SSL Mode : " + config.get(PARAM_SSL_MODE).get(PARAM_MODE).asText()); + additionalParameters.putAll(obtainConnectionOptions(config.get(PARAM_SSL_MODE))); + } else { + additionalParameters.put(PARAM_SSLMODE, REQUIRE); + } + } + additionalParameters.put(ENCRYPTDATA, ENCRYPTDATA_ON); + return additionalParameters; + } + private Map obtainConnectionOptions(final JsonNode encryption) { final Map additionalParameters = new HashMap<>(); if (!encryption.isNull()) { @@ -139,7 +119,6 @@ public JsonNode toJdbcConfig(final JsonNode config) { if (config.has(JdbcUtils.PASSWORD_KEY)) { configBuilder.put(JdbcUtils.PASSWORD_KEY, config.get(JdbcUtils.PASSWORD_KEY).asText()); ->>>>>>> master } if (config.has(JdbcUtils.JDBC_URL_PARAMS_KEY)) { diff --git a/docs/integrations/destinations/teradata.md b/docs/integrations/destinations/teradata.md index e3d5cc81aa52d..6e36c00431144 100644 --- a/docs/integrations/destinations/teradata.md +++ b/docs/integrations/destinations/teradata.md @@ -84,19 +84,10 @@ You can also use a pre-existing user but we highly recommend creating a dedicate ## CHANGELOG -<<<<<<< HEAD | Version | Date | Pull Request | Subject | | :------ | :--------- | :---------------------------------------------- | :---------------------------------------- | | 0.1.4 | 2023-12-04 | https://github.com/airbytehq/airbyte/pull/28667 | Make connector available on Airbyte Cloud | | 0.1.3 | 2023-08-17 | https://github.com/airbytehq/airbyte/pull/30740 | Enable custom DBT transformation | | 0.1.2 | 2023-08-09 | https://github.com/airbytehq/airbyte/pull/29174 | Small internal refactor | | 0.1.1 | 2023-03-03 | https://github.com/airbytehq/airbyte/pull/21760 | Added SSL support | -| 0.1.0 | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428 | New Destination Teradata Vantage | -======= -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :---------------------------------------------- | :------------------------------- | -| 0.1.3 | 2023-08-17 | https://github.com/airbytehq/airbyte/pull/30740 | Enable custom DBT transformation | -| 0.1.2 | 2023-08-09 | https://github.com/airbytehq/airbyte/pull/29174 | Small internal refactor | -| 0.1.1 | 2023-03-03 | https://github.com/airbytehq/airbyte/pull/21760 | Added SSL support | -| 0.1.0 | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428 | New Destination Teradata Vantage | ->>>>>>> master +| 0.1.0 | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428 | New Destination Teradata Vantage | \ No newline at end of file From f8ffc90274db197111c4f75b77a1bb126f9bed2c Mon Sep 17 00:00:00 2001 From: SatishChGit Date: Tue, 19 Dec 2023 16:13:20 +0530 Subject: [PATCH 4/8] Teradata cloud release - Addressed review comment about docker (#33619) --- .../destination-teradata/Dockerfile | 26 ------------------- 1 file changed, 26 deletions(-) delete mode 100644 airbyte-integrations/connectors/destination-teradata/Dockerfile diff --git a/airbyte-integrations/connectors/destination-teradata/Dockerfile b/airbyte-integrations/connectors/destination-teradata/Dockerfile deleted file mode 100644 index 96d74f9054edb..0000000000000 --- a/airbyte-integrations/connectors/destination-teradata/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte -ENV APPLICATION destination-teradata - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION destination-teradata - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.4 -LABEL io.airbyte.name=airbyte/destination-teradata From aedada0978eca462c7410181af0045aca0144efc Mon Sep 17 00:00:00 2001 From: evantahler Date: Tue, 19 Dec 2023 16:45:19 -0800 Subject: [PATCH 5/8] format --- .../teradata/TeradataDestination.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java index b8bf7ec01a197..55aa93c237b4b 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java +++ b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java @@ -69,18 +69,18 @@ private static void createCertificateFile(String fileName, String fileValue) thr @Override protected Map getDefaultConnectionProperties(final JsonNode config) { - final Map additionalParameters = new HashMap<>(); - if (config.has(PARAM_SSL) && config.get(PARAM_SSL).asBoolean()) { - LOGGER.debug("SSL Enabled"); - if (config.has(PARAM_SSL_MODE)) { - LOGGER.debug("Selected SSL Mode : " + config.get(PARAM_SSL_MODE).get(PARAM_MODE).asText()); - additionalParameters.putAll(obtainConnectionOptions(config.get(PARAM_SSL_MODE))); - } else { - additionalParameters.put(PARAM_SSLMODE, REQUIRE); - } - } - additionalParameters.put(ENCRYPTDATA, ENCRYPTDATA_ON); - return additionalParameters; + final Map additionalParameters = new HashMap<>(); + if (config.has(PARAM_SSL) && config.get(PARAM_SSL).asBoolean()) { + LOGGER.debug("SSL Enabled"); + if (config.has(PARAM_SSL_MODE)) { + LOGGER.debug("Selected SSL Mode : " + config.get(PARAM_SSL_MODE).get(PARAM_MODE).asText()); + additionalParameters.putAll(obtainConnectionOptions(config.get(PARAM_SSL_MODE))); + } else { + additionalParameters.put(PARAM_SSLMODE, REQUIRE); + } + } + additionalParameters.put(ENCRYPTDATA, ENCRYPTDATA_ON); + return additionalParameters; } private Map obtainConnectionOptions(final JsonNode encryption) { From b42668c5aa87f75245fedcf2758d421ec196035e Mon Sep 17 00:00:00 2001 From: SatishChGit Date: Fri, 12 Jan 2024 22:19:06 +0530 Subject: [PATCH 6/8] Destination Teradata: Added Primary Index on _airbyte_ab_id to fix NoPI issue (#33872) --- .../destination-teradata/metadata.yaml | 2 +- .../teradata/TeradataSqlOperations.java | 6 +++--- docs/integrations/destinations/teradata.md | 17 +++++++++-------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/destination-teradata/metadata.yaml b/airbyte-integrations/connectors/destination-teradata/metadata.yaml index 17e0de4e04939..27bdb0fa57b39 100644 --- a/airbyte-integrations/connectors/destination-teradata/metadata.yaml +++ b/airbyte-integrations/connectors/destination-teradata/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 58e6f9da-904e-11ed-a1eb-0242ac120002 - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 dockerRepository: airbyte/destination-teradata githubIssueLabel: destination-teradata icon: teradata.svg diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java index 85cf7dc27d53c..77ea8de4ebb9c 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java +++ b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java @@ -107,10 +107,10 @@ public void createTableIfNotExists(final JdbcDatabase database, final String sch @Override public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { return String.format( - "CREATE SET TABLE %s.%s, FALLBACK ( \n" + "%s VARCHAR(256), \n" + "%s JSON, \n" + "%s TIMESTAMP(6) \n" - + ");\n", + "CREATE SET TABLE %s.%s, FALLBACK ( %s VARCHAR(256), %s JSON, %s TIMESTAMP(6)) " + + " UNIQUE PRIMARY INDEX (%s) ", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, - JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + JavaBaseConstants.COLUMN_NAME_EMITTED_AT, JavaBaseConstants.COLUMN_NAME_AB_ID); } @Override diff --git a/docs/integrations/destinations/teradata.md b/docs/integrations/destinations/teradata.md index 6e36c00431144..8f6bfd22c0f2b 100644 --- a/docs/integrations/destinations/teradata.md +++ b/docs/integrations/destinations/teradata.md @@ -26,7 +26,7 @@ You'll need the following information to configure the Teradata destination: Each stream will be output into its own table in Teradata. Each table will contain 3 columns: -- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in Teradata is `VARCHAR(256)`. +- `_airbyte_ab_id`: a unique uuid assigned by Airbyte to each event that is processed. This is the primary index column. The column type in Teradata is `VARCHAR(256)`. - `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in Teradata is `TIMESTAMP(6)`. - `_airbyte_data`: a json blob representing with the event data. The column type in Teradata is `JSON`. @@ -84,10 +84,11 @@ You can also use a pre-existing user but we highly recommend creating a dedicate ## CHANGELOG -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :---------------------------------------------- | :---------------------------------------- | -| 0.1.4 | 2023-12-04 | https://github.com/airbytehq/airbyte/pull/28667 | Make connector available on Airbyte Cloud | -| 0.1.3 | 2023-08-17 | https://github.com/airbytehq/airbyte/pull/30740 | Enable custom DBT transformation | -| 0.1.2 | 2023-08-09 | https://github.com/airbytehq/airbyte/pull/29174 | Small internal refactor | -| 0.1.1 | 2023-03-03 | https://github.com/airbytehq/airbyte/pull/21760 | Added SSL support | -| 0.1.0 | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428 | New Destination Teradata Vantage | \ No newline at end of file +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :---------------------------------------------- |:--------------------------------------------------------| +| 0.1.5 | 2024-01-12 | https://github.com/airbytehq/airbyte/pull/33872 | Added Primary Index on _airbyte_ab_id to fix NoPI issue | +| 0.1.4 | 2023-12-04 | https://github.com/airbytehq/airbyte/pull/28667 | Make connector available on Airbyte Cloud | +| 0.1.3 | 2023-08-17 | https://github.com/airbytehq/airbyte/pull/30740 | Enable custom DBT transformation | +| 0.1.2 | 2023-08-09 | https://github.com/airbytehq/airbyte/pull/29174 | Small internal refactor | +| 0.1.1 | 2023-03-03 | https://github.com/airbytehq/airbyte/pull/21760 | Added SSL support | +| 0.1.0 | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428 | New Destination Teradata Vantage | \ No newline at end of file From 63e9dfc877b26106fe29ad97682bd330a51f263a Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Tue, 30 Jan 2024 13:35:49 -0300 Subject: [PATCH 7/8] format --- .../destination/teradata/TeradataSqlOperations.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java index 77ea8de4ebb9c..55522de02b66b 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java +++ b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java @@ -108,7 +108,7 @@ public void createTableIfNotExists(final JdbcDatabase database, final String sch public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { return String.format( "CREATE SET TABLE %s.%s, FALLBACK ( %s VARCHAR(256), %s JSON, %s TIMESTAMP(6)) " + - " UNIQUE PRIMARY INDEX (%s) ", + " UNIQUE PRIMARY INDEX (%s) ", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_EMITTED_AT, JavaBaseConstants.COLUMN_NAME_AB_ID); } From 7bc01a6b214cb53bfdd497e4f7529dbc1419b167 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Tue, 30 Jan 2024 16:41:52 -0300 Subject: [PATCH 8/8] disbale normalization --- .../connectors/destination-teradata/metadata.yaml | 1 - .../teradata/TeradataDestinationAcceptanceTest.java | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-teradata/metadata.yaml b/airbyte-integrations/connectors/destination-teradata/metadata.yaml index 27bdb0fa57b39..de975748835f7 100644 --- a/airbyte-integrations/connectors/destination-teradata/metadata.yaml +++ b/airbyte-integrations/connectors/destination-teradata/metadata.yaml @@ -15,7 +15,6 @@ data: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/teradata - supportsDbt: true tags: - language:java ab_internal: diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java index c3fa9274ad986..ea6969b8c4400 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java @@ -179,6 +179,12 @@ public void testSecondSync() { // overrides test in coming releases } + @Override + @Test + public void testCustomDbtTransformations() throws Exception { + // overrides test in coming releases + } + protected DataSource getDataSource(final JsonNode config) { final JsonNode jdbcConfig = destination.toJdbcConfig(config); return DataSourceFactory.create(jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText(),