From 3d882075a750fc55ea8fad89d5e8fabd7034db6c Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Tue, 30 Jan 2024 19:43:34 -0300 Subject: [PATCH] Destination Teradata: make connector avaialble on Airbyte Cloud (#28667) Co-authored-by: SatishChGit Co-authored-by: evantahler --- .../destination-teradata/metadata.yaml | 5 ++--- .../teradata/TeradataDestination.java | 17 +++++++++++------ .../teradata/TeradataSqlOperations.java | 6 +++--- .../TeradataDestinationAcceptanceTest.java | 6 ++++++ docs/integrations/destinations/teradata.md | 16 +++++++++------- 5 files changed, 31 insertions(+), 19 deletions(-) diff --git a/airbyte-integrations/connectors/destination-teradata/metadata.yaml b/airbyte-integrations/connectors/destination-teradata/metadata.yaml index 91738294ff4e3..de975748835f7 100644 --- a/airbyte-integrations/connectors/destination-teradata/metadata.yaml +++ b/airbyte-integrations/connectors/destination-teradata/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 58e6f9da-904e-11ed-a1eb-0242ac120002 - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.5 dockerRepository: airbyte/destination-teradata githubIssueLabel: destination-teradata icon: teradata.svg @@ -10,12 +10,11 @@ data: name: Teradata Vantage registries: cloud: - enabled: false + enabled: true oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/teradata - supportsDbt: true tags: - language:java ab_internal: diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java index 37fd84a973a66..55aa93c237b4b 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java +++ b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java @@ -49,6 +49,10 @@ public class TeradataDestination extends AbstractJdbcDestination implements Dest protected static final String CA_CERT_KEY = "ssl_ca_certificate"; + protected static final String ENCRYPTDATA = "ENCRYPTDATA"; + + protected static final String ENCRYPTDATA_ON = "ON"; + public static void main(String[] args) throws Exception { new IntegrationRunner(new TeradataDestination()).run(args); } @@ -57,6 +61,12 @@ public TeradataDestination() { super(DRIVER_CLASS, new StandardNameTransformer(), new TeradataSqlOperations()); } + private static void createCertificateFile(String fileName, String fileValue) throws IOException { + try (final PrintWriter out = new PrintWriter(fileName, StandardCharsets.UTF_8)) { + out.print(fileValue); + } + } + @Override protected Map getDefaultConnectionProperties(final JsonNode config) { final Map additionalParameters = new HashMap<>(); @@ -69,15 +79,10 @@ protected Map getDefaultConnectionProperties(final JsonNode conf additionalParameters.put(PARAM_SSLMODE, REQUIRE); } } + additionalParameters.put(ENCRYPTDATA, ENCRYPTDATA_ON); return additionalParameters; } - private static void createCertificateFile(String fileName, String fileValue) throws IOException { - try (final PrintWriter out = new PrintWriter(fileName, StandardCharsets.UTF_8)) { - out.print(fileValue); - } - } - private Map obtainConnectionOptions(final JsonNode encryption) { final Map additionalParameters = new HashMap<>(); if (!encryption.isNull()) { diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java index 85cf7dc27d53c..55522de02b66b 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java +++ b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java @@ -107,10 +107,10 @@ public void createTableIfNotExists(final JdbcDatabase database, final String sch @Override public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { return String.format( - "CREATE SET TABLE %s.%s, FALLBACK ( \n" + "%s VARCHAR(256), \n" + "%s JSON, \n" + "%s TIMESTAMP(6) \n" - + ");\n", + "CREATE SET TABLE %s.%s, FALLBACK ( %s VARCHAR(256), %s JSON, %s TIMESTAMP(6)) " + + " UNIQUE PRIMARY INDEX (%s) ", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, - JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + JavaBaseConstants.COLUMN_NAME_EMITTED_AT, JavaBaseConstants.COLUMN_NAME_AB_ID); } @Override diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java index c3fa9274ad986..ea6969b8c4400 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java @@ -179,6 +179,12 @@ public void testSecondSync() { // overrides test in coming releases } + @Override + @Test + public void testCustomDbtTransformations() throws Exception { + // overrides test in coming releases + } + protected DataSource getDataSource(final JsonNode config) { final JsonNode jdbcConfig = destination.toJdbcConfig(config); return DataSourceFactory.create(jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText(), diff --git a/docs/integrations/destinations/teradata.md b/docs/integrations/destinations/teradata.md index 74f2b89fd598b..8f6bfd22c0f2b 100644 --- a/docs/integrations/destinations/teradata.md +++ b/docs/integrations/destinations/teradata.md @@ -26,7 +26,7 @@ You'll need the following information to configure the Teradata destination: Each stream will be output into its own table in Teradata. Each table will contain 3 columns: -- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in Teradata is `VARCHAR(256)`. +- `_airbyte_ab_id`: a unique uuid assigned by Airbyte to each event that is processed. This is the primary index column. The column type in Teradata is `VARCHAR(256)`. - `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in Teradata is `TIMESTAMP(6)`. - `_airbyte_data`: a json blob representing with the event data. The column type in Teradata is `JSON`. @@ -84,9 +84,11 @@ You can also use a pre-existing user but we highly recommend creating a dedicate ## CHANGELOG -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :---------------------------------------------- | :------------------------------- | -| 0.1.3 | 2023-08-17 | https://github.com/airbytehq/airbyte/pull/30740 | Enable custom DBT transformation | -| 0.1.2 | 2023-08-09 | https://github.com/airbytehq/airbyte/pull/29174 | Small internal refactor | -| 0.1.1 | 2023-03-03 | https://github.com/airbytehq/airbyte/pull/21760 | Added SSL support | -| 0.1.0 | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428 | New Destination Teradata Vantage | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :---------------------------------------------- |:--------------------------------------------------------| +| 0.1.5 | 2024-01-12 | https://github.com/airbytehq/airbyte/pull/33872 | Added Primary Index on _airbyte_ab_id to fix NoPI issue | +| 0.1.4 | 2023-12-04 | https://github.com/airbytehq/airbyte/pull/28667 | Make connector available on Airbyte Cloud | +| 0.1.3 | 2023-08-17 | https://github.com/airbytehq/airbyte/pull/30740 | Enable custom DBT transformation | +| 0.1.2 | 2023-08-09 | https://github.com/airbytehq/airbyte/pull/29174 | Small internal refactor | +| 0.1.1 | 2023-03-03 | https://github.com/airbytehq/airbyte/pull/21760 | Added SSL support | +| 0.1.0 | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428 | New Destination Teradata Vantage | \ No newline at end of file