From 38e6bdf494f1bcd1d9ecb8a68713bd1b77aa4fe9 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Tue, 14 Dec 2021 15:58:57 +0200 Subject: [PATCH 1/7] Destination Snowflake update check method to verify permission for stages --- .../SnowflakeInternalStagingDestination.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java index bf32c15e8c2f..abbebb445338 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java @@ -10,16 +10,49 @@ import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.UUID; import java.util.function.Consumer; public class SnowflakeInternalStagingDestination extends AbstractJdbcDestination implements Destination { + private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeInternalStagingDestination.class); + public SnowflakeInternalStagingDestination() { super("", new SnowflakeSQLNameTransformer(), new SnowflakeStagingSqlOperations()); } + @Override + public AirbyteConnectionStatus check(JsonNode config) { + SnowflakeSQLNameTransformer nameTransformer = new SnowflakeSQLNameTransformer(); + SnowflakeStagingSqlOperations snowflakeStagingSqlOperations = new SnowflakeStagingSqlOperations(); + try (final JdbcDatabase database = getDatabase(config)) { + final String outputSchema = super.getNamingResolver().getIdentifier(config.get("schema").asText()); + attemptSQLCreateAndDropTableOperations(outputSchema, database, nameTransformer, snowflakeStagingSqlOperations); + attemptSQLCreateAndDropStages(outputSchema, database, nameTransformer, snowflakeStagingSqlOperations); + return new AirbyteConnectionStatus().withStatus(AirbyteConnectionStatus.Status.SUCCEEDED); + } catch (final Exception e) { + LOGGER.error("Exception while checking connection: ", e); + return new AirbyteConnectionStatus() + .withStatus(AirbyteConnectionStatus.Status.FAILED) + .withMessage("Could not connect with provided configuration. \n" + e.getMessage()); + } + } + + private static void attemptSQLCreateAndDropStages(String outputSchema, JdbcDatabase database, SnowflakeSQLNameTransformer namingResolver, SnowflakeStagingSqlOperations sqlOperations) throws Exception { + + // verify we have permissions to create/drop stage + final String outputTableName = namingResolver.getIdentifier("_airbyte_connection_test_" + UUID.randomUUID().toString().replaceAll("-", "")); + String stageName = namingResolver.getStageName(outputSchema, outputTableName);; + sqlOperations.createStageIfNotExists(database, stageName); + sqlOperations.dropStageIfExists(database,stageName); + } + @Override protected JdbcDatabase getDatabase(final JsonNode config) { return SnowflakeDatabase.getDatabase(config); From 93d53ac478146d7c5c5b7ef62585c90a4021dd83 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Wed, 15 Dec 2021 10:45:04 +0200 Subject: [PATCH 2/7] fix for jdk 17 --- .../integrations/destination/snowflake/SnowflakeDatabase.java | 2 ++ docs/integrations/destinations/snowflake.md | 3 +++ 2 files changed, 5 insertions(+) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java index c484a78452af..a1548c162373 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java @@ -42,6 +42,8 @@ public static Connection getConnection(final JsonNode config) throws SQLExceptio // https://docs.snowflake.com/en/user-guide/jdbc-parameters.html#application // identify airbyte traffic to snowflake to enable partnership & optimization opportunities properties.put("application", "airbyte"); + // Needed for JDK17 - see https://stackoverflow.com/questions/67409650/snowflake-jdbc-driver-internal-error-fail-to-retrieve-row-count-for-first-arrow + properties.put("JDBC_QUERY_RESULT_FORMAT", "JSON"); return DriverManager.getConnection(connectUrl, properties); } diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index b84bbc84f841..74e4db39efbb 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -152,6 +152,8 @@ By default, Airbyte uses batches of `INSERT` commands to add data to a temporary Internal named stages are storage location objects within a Snowflake database/schema. Because they are database objects, the same security permissions apply as with any other database objects. No need to provide additional properties for internal staging +**Operating on a stage also requires the USAGE privilege on the parent database and schema.** + ### AWS S3 For AWS S3, you will need to create a bucket and provide credentials to access the bucket. We recommend creating a bucket that is only used for Airbyte to stage data to Snowflake. Airbyte needs read/write access to interact with this bucket. @@ -194,6 +196,7 @@ Finally, you need to add read/write permissions to your bucket with that email. | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.3.21 | 2021-12-15 | [#8781](https://github.com/airbytehq/airbyte/pull/8781) | Updated check method to verify permissions to create/drop stage for internal staging; compatibility fix for Java 17 | | 0.3.20 | 2021-12-10 | [#8562](https://github.com/airbytehq/airbyte/pull/8562) | Moving classes around for better dependency management; compatibility fix for Java 17 | | 0.3.19 | 2021-12-06 | [#8528](https://github.com/airbytehq/airbyte/pull/8528) | Set Internal Staging as default choice | | 0.3.18 | 2021-11-26 | [#8253](https://github.com/airbytehq/airbyte/pull/8253) | Snowflake Internal Staging Support | From cba848cf3ca92c8b9b81c61946e01e4d01e6f620 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Wed, 15 Dec 2021 10:47:04 +0200 Subject: [PATCH 3/7] fix for jdk 17 --- .../integrations/destination/snowflake/SnowflakeDatabase.java | 2 ++ docs/integrations/destinations/snowflake.md | 3 +++ 2 files changed, 5 insertions(+) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java index c484a78452af..a1548c162373 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java @@ -42,6 +42,8 @@ public static Connection getConnection(final JsonNode config) throws SQLExceptio // https://docs.snowflake.com/en/user-guide/jdbc-parameters.html#application // identify airbyte traffic to snowflake to enable partnership & optimization opportunities properties.put("application", "airbyte"); + // Needed for JDK17 - see https://stackoverflow.com/questions/67409650/snowflake-jdbc-driver-internal-error-fail-to-retrieve-row-count-for-first-arrow + properties.put("JDBC_QUERY_RESULT_FORMAT", "JSON"); return DriverManager.getConnection(connectUrl, properties); } diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index b84bbc84f841..74e4db39efbb 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -152,6 +152,8 @@ By default, Airbyte uses batches of `INSERT` commands to add data to a temporary Internal named stages are storage location objects within a Snowflake database/schema. Because they are database objects, the same security permissions apply as with any other database objects. No need to provide additional properties for internal staging +**Operating on a stage also requires the USAGE privilege on the parent database and schema.** + ### AWS S3 For AWS S3, you will need to create a bucket and provide credentials to access the bucket. We recommend creating a bucket that is only used for Airbyte to stage data to Snowflake. Airbyte needs read/write access to interact with this bucket. @@ -194,6 +196,7 @@ Finally, you need to add read/write permissions to your bucket with that email. | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.3.21 | 2021-12-15 | [#8781](https://github.com/airbytehq/airbyte/pull/8781) | Updated check method to verify permissions to create/drop stage for internal staging; compatibility fix for Java 17 | | 0.3.20 | 2021-12-10 | [#8562](https://github.com/airbytehq/airbyte/pull/8562) | Moving classes around for better dependency management; compatibility fix for Java 17 | | 0.3.19 | 2021-12-06 | [#8528](https://github.com/airbytehq/airbyte/pull/8528) | Set Internal Staging as default choice | | 0.3.18 | 2021-11-26 | [#8253](https://github.com/airbytehq/airbyte/pull/8253) | Snowflake Internal Staging Support | From 578b106b9fe3170ef69b4ef548f60616bf921f6c Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Wed, 15 Dec 2021 11:24:29 +0200 Subject: [PATCH 4/7] fix with ci secrets --- tools/bin/ci_credentials.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index bdf0e35d3b9e..73588a0de15e 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -169,8 +169,8 @@ read_secrets destination-dynamodb "$DESTINATION_DYNAMODB_TEST_CREDS" read_secrets destination-oracle "$AWS_ORACLE_INTEGRATION_TEST_CREDS" read_secrets destination-s3 "$DESTINATION_S3_INTEGRATION_TEST_CREDS" read_secrets destination-azure-blob-storage "$DESTINATION_AZURE_BLOB_CREDS" -read_secrets destination-snowflake "$SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS" "copy_gcs_config.json" -read_secrets destination-snowflake "$SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS" "copy_s3_config.json" +#read_secrets destination-snowflake "$SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS" "copy_gcs_config.json" +#read_secrets destination-snowflake "$SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS" "copy_s3_config.json" read_secrets destination-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "insert_config.json" read_secrets base-normalization "$BIGQUERY_INTEGRATION_TEST_CREDS" "bigquery.json" From 91a749504f7716b8622c9235a9a5242446bb51e5 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Wed, 15 Dec 2021 11:31:16 +0200 Subject: [PATCH 5/7] fix with ci secrets --- tools/bin/ci_credentials.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 73588a0de15e..076fb4d0d188 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -171,7 +171,7 @@ read_secrets destination-s3 "$DESTINATION_S3_INTEGRATION_TEST_CREDS" read_secrets destination-azure-blob-storage "$DESTINATION_AZURE_BLOB_CREDS" #read_secrets destination-snowflake "$SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS" "copy_gcs_config.json" #read_secrets destination-snowflake "$SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS" "copy_s3_config.json" -read_secrets destination-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "insert_config.json" +#read_secrets destination-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "insert_config.json" read_secrets base-normalization "$BIGQUERY_INTEGRATION_TEST_CREDS" "bigquery.json" read_secrets base-normalization "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "snowflake.json" From 7dde0f21b9aaa9407207518aacb4dd6cf8b30b81 Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Wed, 15 Dec 2021 12:46:04 +0200 Subject: [PATCH 6/7] removed snowflake secrets from ci_credentials.sh --- tools/bin/ci_credentials.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 076fb4d0d188..bb8cb8319e03 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -169,9 +169,6 @@ read_secrets destination-dynamodb "$DESTINATION_DYNAMODB_TEST_CREDS" read_secrets destination-oracle "$AWS_ORACLE_INTEGRATION_TEST_CREDS" read_secrets destination-s3 "$DESTINATION_S3_INTEGRATION_TEST_CREDS" read_secrets destination-azure-blob-storage "$DESTINATION_AZURE_BLOB_CREDS" -#read_secrets destination-snowflake "$SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS" "copy_gcs_config.json" -#read_secrets destination-snowflake "$SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS" "copy_s3_config.json" -#read_secrets destination-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "insert_config.json" read_secrets base-normalization "$BIGQUERY_INTEGRATION_TEST_CREDS" "bigquery.json" read_secrets base-normalization "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "snowflake.json" From 4d19b177354a230ab5ad97349d3c549d5ca3950b Mon Sep 17 00:00:00 2001 From: vmaltsev Date: Thu, 16 Dec 2021 10:49:30 +0200 Subject: [PATCH 7/7] bump version --- .../424892c4-daac-4491-b35d-c6688ba547ba.json | 2 +- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../connectors/destination-snowflake/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json index e748337dada5..20d735eb73a1 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json @@ -2,7 +2,7 @@ "destinationDefinitionId": "424892c4-daac-4491-b35d-c6688ba547ba", "name": "Snowflake", "dockerRepository": "airbyte/destination-snowflake", - "dockerImageTag": "0.3.19", + "dockerImageTag": "0.3.21", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/snowflake", "icon": "snowflake.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 32c6a5218e5f..8a1071f10d16 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -173,7 +173,7 @@ - name: Snowflake destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba dockerRepository: airbyte/destination-snowflake - dockerImageTag: 0.3.20 + dockerImageTag: 0.3.21 documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake icon: snowflake.svg - name: MariaDB ColumnStore diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index 827233b3d5fd..e80a5fe6a0af 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -18,5 +18,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.20 +LABEL io.airbyte.version=0.3.21 LABEL io.airbyte.name=airbyte/destination-snowflake