From 15fce4f54ee8efe4c0afa4cf88d42c4752e0950a Mon Sep 17 00:00:00 2001 From: Eugene Date: Tue, 31 Jan 2023 15:54:20 +0200 Subject: [PATCH] Destination-s3: updated test and documentation - set minimum required policies (#22058) * [21331] Destination-s3: updated documentation - set minimum required policies --- .../s3/S3AvroDestinationAcceptanceTest.java | 6 ++++++ .../s3/S3CsvDestinationAcceptanceTest.java | 6 ++++++ .../S3CsvGzipDestinationAcceptanceTest.java | 6 ++++++ .../s3/S3DestinationTestUtils.java | 20 +++++++++++++++++++ .../s3/S3JsonlDestinationAcceptanceTest.java | 6 ++++++ .../S3JsonlGzipDestinationAcceptanceTest.java | 6 ++++++ .../S3ParquetDestinationAcceptanceTest.java | 6 ++++++ docs/integrations/destinations/s3.md | 15 +++++++++++--- 8 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3DestinationTestUtils.java diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3AvroDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3AvroDestinationAcceptanceTest.java index b006233b3f8e8..b94261abd5efa 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3AvroDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3AvroDestinationAcceptanceTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.destination.s3; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.standardtest.destination.ProtocolVersion; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; @@ -19,4 +20,9 @@ protected TestDataComparator getTestDataComparator() { return new S3AvroParquetTestDataComparator(); } + @Override + protected JsonNode getBaseConfigJson() { + return S3DestinationTestUtils.getBaseConfigJsonFilePath(); + } + } diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java index a3b82902e543e..5e52653adbc58 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.destination.s3; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.standardtest.destination.ProtocolVersion; public class S3CsvDestinationAcceptanceTest extends S3BaseCsvDestinationAcceptanceTest { @@ -13,4 +14,9 @@ public ProtocolVersion getProtocolVersion() { return ProtocolVersion.V1; } + @Override + protected JsonNode getBaseConfigJson() { + return S3DestinationTestUtils.getBaseConfigJsonFilePath(); + } + } diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvGzipDestinationAcceptanceTest.java index fe5e33b84b75d..fb4b6cdc41537 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvGzipDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvGzipDestinationAcceptanceTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.destination.s3; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.standardtest.destination.ProtocolVersion; public class S3CsvGzipDestinationAcceptanceTest extends S3BaseCsvGzipDestinationAcceptanceTest { @@ -13,4 +14,9 @@ public ProtocolVersion getProtocolVersion() { return ProtocolVersion.V1; } + @Override + protected JsonNode getBaseConfigJson() { + return S3DestinationTestUtils.getBaseConfigJsonFilePath(); + } + } diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3DestinationTestUtils.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3DestinationTestUtils.java new file mode 100644 index 0000000000000..35429abbb8670 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3DestinationTestUtils.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.s3; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import java.nio.file.Path; + +public class S3DestinationTestUtils { + + private static final String SECRET_PATH = "secrets/s3_dest_min_required_permissions_creds.json"; + + public static JsonNode getBaseConfigJsonFilePath() { + return Jsons.deserialize(IOs.readFile(Path.of(SECRET_PATH))); + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlDestinationAcceptanceTest.java index 66424315cfcc9..6a92008b9eb52 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlDestinationAcceptanceTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.destination.s3; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.standardtest.destination.ProtocolVersion; public class S3JsonlDestinationAcceptanceTest extends S3BaseJsonlDestinationAcceptanceTest { @@ -13,4 +14,9 @@ public ProtocolVersion getProtocolVersion() { return ProtocolVersion.V1; } + @Override + protected JsonNode getBaseConfigJson() { + return S3DestinationTestUtils.getBaseConfigJsonFilePath(); + } + } diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlGzipDestinationAcceptanceTest.java index 59f7149b2c166..8c606f507a7ab 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlGzipDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlGzipDestinationAcceptanceTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.destination.s3; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.standardtest.destination.ProtocolVersion; public class S3JsonlGzipDestinationAcceptanceTest extends S3BaseJsonlGzipDestinationAcceptanceTest { @@ -13,4 +14,9 @@ public ProtocolVersion getProtocolVersion() { return ProtocolVersion.V1; } + @Override + protected JsonNode getBaseConfigJson() { + return S3DestinationTestUtils.getBaseConfigJsonFilePath(); + } + } diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3ParquetDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3ParquetDestinationAcceptanceTest.java index 113061ce10bb8..e336f6294e2d4 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3ParquetDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3ParquetDestinationAcceptanceTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.destination.s3; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.standardtest.destination.ProtocolVersion; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; @@ -19,4 +20,9 @@ protected TestDataComparator getTestDataComparator() { return new S3AvroParquetTestDataComparator(); } + @Override + protected JsonNode getBaseConfigJson() { + return S3DestinationTestUtils.getBaseConfigJsonFilePath(); + } + } diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index db26ac102e0d0..faaff76ca4cbc 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -91,19 +91,28 @@ NOTE: If the S3 cluster is not configured to use TLS, the connection to Amazon S 5. Click `Set up destination`. -In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and "S3 Access Key" are used have access to both the bucket and its contents. Policies to use: +In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and "S3 Access Key" are used have access to both the bucket and its contents. Minimum required Policies to use: ```json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", - "Action": "s3:*", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:PutObjectAcl", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:AbortMultipartUpload", + "s3:GetBucketLocation" + ], "Resource": [ "arn:aws:s3:::YOUR_BUCKET_NAME/*", "arn:aws:s3:::YOUR_BUCKET_NAME" ] - } + } ] } ```