diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json index a3384e94df6d..77954f0c5ee3 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json @@ -2,7 +2,7 @@ "destinationDefinitionId": "f7a7d195-377f-cf5b-70a5-be6b819019dc", "name": "Redshift", "dockerRepository": "airbyte/destination-redshift", - "dockerImageTag": "0.3.18", + "dockerImageTag": "0.3.19", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redshift", "icon": "redshift.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json index 1d16d1f8c226..82a5855d41fe 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e87ffa8e-a3b5-f69c-9076-6011339de1f6.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "e87ffa8e-a3b5-f69c-9076-6011339de1f6", "name": "Redshift", "dockerRepository": "airbyte/source-redshift", - "dockerImageTag": "0.3.3", + "dockerImageTag": "0.3.4", "documentationUrl": "https://docs.airbyte.io/integrations/sources/redshift", "icon": "redshift.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 8426c2a52fd1..1b306a02917d 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -52,7 +52,7 @@ - destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc name: Redshift dockerRepository: airbyte/destination-redshift - dockerImageTag: 0.3.18 + dockerImageTag: 0.3.19 documentationUrl: https://docs.airbyte.io/integrations/destinations/redshift icon: redshift.svg - destinationDefinitionId: af7c921e-5892-4ff2-b6c1-4a5ab258fb7e diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index a7a5145c6184..e2df1dc60ff7 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -180,7 +180,7 @@ - sourceDefinitionId: e87ffa8e-a3b5-f69c-9076-6011339de1f6 name: Redshift dockerRepository: airbyte/source-redshift - dockerImageTag: 0.3.3 + dockerImageTag: 0.3.4 documentationUrl: https://docs.airbyte.io/integrations/sources/redshift icon: redshift.svg sourceType: database diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index e69b69767764..2f733ac12580 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.18 +LABEL io.airbyte.version=0.3.19 LABEL io.airbyte.name=airbyte/destination-redshift diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java index 21f8d8585fda..8311463a35e7 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java @@ -37,26 +37,21 @@ public JdbcDatabase getDatabase(final JsonNode config) { return getJdbcDatabase(config); } - private static void readSsl(final JsonNode redshiftConfig, final List additionalProperties) { - final boolean tls = redshiftConfig.has("tls") && redshiftConfig.get("tls").asBoolean(); // for backward compatibility - if (!tls) { - additionalProperties.add("ssl=false"); - } else { - additionalProperties.add("ssl=true"); - additionalProperties.add("sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); - } + private static void addSsl(final List additionalProperties) { + additionalProperties.add("ssl=true"); + additionalProperties.add("sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); } public static JdbcDatabase getJdbcDatabase(final JsonNode config) { final List additionalProperties = new ArrayList<>(); final var jdbcConfig = RedshiftInsertDestination.getJdbcConfig(config); - readSsl(config, additionalProperties); + addSsl(additionalProperties); return Databases.createJdbcDatabase( jdbcConfig.get("username").asText(), jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null, jdbcConfig.get("jdbc_url").asText(), RedshiftInsertDestination.DRIVER_CLASS, - additionalProperties.isEmpty() ? "" : String.join(";", additionalProperties)); + String.join(";", additionalProperties)); } public static JsonNode getJdbcConfig(final JsonNode redshiftConfig) { diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json index c4f0c1860e39..cade59598ca9 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json @@ -105,12 +105,6 @@ "examples": ["10"], "description": "Optional. Increase this if syncing tables larger than 100GB. Only relevant for COPY. Files are streamed to S3 in parts. This determines the size of each part, in MBs. As S3 has a limit of 10,000 parts per file, part size affects the table size. This is 10MB by default, resulting in a default limit of 100GB tables. Note, a larger part size will result in larger memory requirements. A rule of thumb is to multiply the part size by 10 to get the memory requirement. Modify this with care.", "title": "Stream Part Size" - }, - "tls": { - "title": "TLS connection", - "type": "boolean", - "description": "Indicates whether TLS encryption protocol will be used to connect to Redshift. It is recommended to use TLS connection if possible", - "default": true } } } diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java index 023d416997c9..44d232a526b0 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftCopyDestinationAcceptanceTest.java @@ -142,7 +142,8 @@ protected Database getDatabase() { baseConfig.get("host").asText(), baseConfig.get("port").asText(), baseConfig.get("database").asText()), - "com.amazon.redshift.jdbc.Driver", null); + "com.amazon.redshift.jdbc.Driver", null, + "ssl=true;sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); } @Override diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftDestinationAcceptanceTestSSL.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftDestinationAcceptanceTestSSL.java deleted file mode 100644 index bdb5e1689ed2..000000000000 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftDestinationAcceptanceTestSSL.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.redshift; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.Databases; - -public class RedshiftDestinationAcceptanceTestSSL extends RedshiftCopyDestinationAcceptanceTest { - - @Override - protected JsonNode getConfig() { - final JsonNode baseConfig = Jsons.clone(getStaticConfig()); - ((ObjectNode) baseConfig).put("tls", true); - ((ObjectNode) baseConfig).put("schema", Strings.addRandomSuffix("integration_test", "_", 5)); - return baseConfig; - } - - @Override - protected Database getDatabase() { - return Databases.createDatabase( - getConfig().get("username").asText(), - getConfig().has("password") ? getConfig().get("password").asText() : null, - String.format("jdbc:redshift://%s:%s/%s", - getConfig().get("host").asText(), - getConfig().get("port").asText(), - getConfig().get("database").asText()), - RedshiftInsertDestination.DRIVER_CLASS, null, - "ssl=true;sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); - } - -} diff --git a/airbyte-integrations/connectors/source-redshift/Dockerfile b/airbyte-integrations/connectors/source-redshift/Dockerfile index 37f100a46af6..90fec9ee4931 100644 --- a/airbyte-integrations/connectors/source-redshift/Dockerfile +++ b/airbyte-integrations/connectors/source-redshift/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.3 +LABEL io.airbyte.version=0.3.4 LABEL io.airbyte.name=airbyte/source-redshift diff --git a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java index f80971a227ee..419e627ca450 100644 --- a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java +++ b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java @@ -37,24 +37,16 @@ public JsonNode toDatabaseConfig(final JsonNode redshiftConfig) { redshiftConfig.get("host").asText(), redshiftConfig.get("port").asText(), redshiftConfig.get("database").asText())); - readSsl(redshiftConfig, additionalProperties); + addSsl(additionalProperties); + builder.put("connection_properties", String.join(";", additionalProperties)); - if (!additionalProperties.isEmpty()) { - final String connectionParams = String.join(";", additionalProperties); - builder.put("connection_properties", connectionParams); - } return Jsons.jsonNode(builder .build()); } - private void readSsl(final JsonNode redshiftConfig, final List additionalProperties) { - final boolean tls = redshiftConfig.has("tls") && redshiftConfig.get("tls").asBoolean(); // for backward compatibility - if (!tls) { - additionalProperties.add("ssl=false"); - } else { - additionalProperties.add("ssl=true"); - additionalProperties.add("sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); - } + private void addSsl(final List additionalProperties) { + additionalProperties.add("ssl=true"); + additionalProperties.add("sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); } @Override diff --git a/airbyte-integrations/connectors/source-redshift/src/main/resources/spec.json b/airbyte-integrations/connectors/source-redshift/src/main/resources/spec.json index c6280363aad9..e1e826d674e2 100644 --- a/airbyte-integrations/connectors/source-redshift/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-redshift/src/main/resources/spec.json @@ -32,12 +32,6 @@ "description": "Password associated with the username.", "type": "string", "airbyte_secret": true - }, - "tls": { - "title": "TLS connection", - "type": "boolean", - "description": "Indicates whether TLS encryption protocol will be used to connect to Redshift. It is recommended to use TLS connection if possible", - "default": true } } } diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index 49f0c4574a59..ed69c1814449 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -105,13 +105,20 @@ Therefore, Airbyte Redshift destination will create tables and schemas using the ## Data Size Limitations Redshift specifies a maximum limit of 65535 bytes to store the raw JSON record data. Thus, when a row is too big to fit, the Redshift destination fails to load such data and currently ignores that record. +## Data Size Limitations +Redshift specifies a maximum limit of 65535 bytes to store the raw JSON record data. Thus, when a row is too big to fit, the Redshift destination fails to load such data and currently ignores that record. See [docs](https://docs.aws.amazon.com/redshift/latest/dg/r_Character_types.html) +## Encryption + +All Redshift connections are encrypted using SSL + ## Changelog | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.3.19 | 2021-10-21 | [7234](https://github.com/airbytehq/airbyte/pull/7234) | Allow SSL traffic only | | 0.3.17 | 2021-10-12 | [6965](https://github.com/airbytehq/airbyte/pull/6965) | Added SSL Support | | 0.3.16 | 2021-10-11 | [6949](https://github.com/airbytehq/airbyte/pull/6949) | Each stream was split into files of 10,000 records each for copying using S3 or GCS | | 0.3.14 | 2021-10-08 | [5924](https://github.com/airbytehq/airbyte/pull/5924) | Fixed AWS S3 Staging COPY is writing records from different table in the same raw table | diff --git a/docs/integrations/sources/redshift.md b/docs/integrations/sources/redshift.md index da891d0ab874..306f9aa5bd96 100644 --- a/docs/integrations/sources/redshift.md +++ b/docs/integrations/sources/redshift.md @@ -45,9 +45,14 @@ This is dependent on your networking setup. The easiest way to verify if Airbyte Next is to provide the necessary information on how to connect to your cluster such as the `host` whcih is part of the connection string or Endpoint accessible [here](https://docs.aws.amazon.com/redshift/latest/gsg/rs-gsg-connect-to-cluster.html#rs-gsg-how-to-get-connection-string) without the `port` and `database` name \(it typically includes the cluster-id, region and end with `.redshift.amazonaws.com`\). +## Encryption + +All Redshift connections are encrypted using SSL + ## Changelog | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.3.4 | 2021-10-21 | [7234](https://github.com/airbytehq/airbyte/pull/7234) | Allow SSL traffic only | | 0.3.3 | 2021-10-12 | [6965](https://github.com/airbytehq/airbyte/pull/6965) | Added SSL Support | | 0.3.2 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | \ No newline at end of file