Destination-s3: updated test and documentation - set minimum required…

… policies (#22058) * [21331] Destination-s3: updated documentation - set minimum required policies
airbytehq · Jan 31, 2023 · 15fce4f · 15fce4f
1 parent 65c0d4a
commit 15fce4f
Show file tree

Hide file tree

Showing 8 changed files with 68 additions and 3 deletions.
diff --git a/...egration/java/io/airbyte/integrations/destination/s3/S3AvroDestinationAcceptanceTest.java b/...egration/java/io/airbyte/integrations/destination/s3/S3AvroDestinationAcceptanceTest.java
@@ -4,6 +4,7 @@
 
 package io.airbyte.integrations.destination.s3;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import io.airbyte.integrations.standardtest.destination.ProtocolVersion;
 import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
 
@@ -19,4 +20,9 @@ protected TestDataComparator getTestDataComparator() {
     return new S3AvroParquetTestDataComparator();
   }
 
+  @Override
+  protected JsonNode getBaseConfigJson() {
+    return S3DestinationTestUtils.getBaseConfigJsonFilePath();
+  }
+
 }
diff --git a/...tegration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java b/...tegration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java
@@ -4,6 +4,7 @@
 
 package io.airbyte.integrations.destination.s3;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import io.airbyte.integrations.standardtest.destination.ProtocolVersion;
 
 public class S3CsvDestinationAcceptanceTest extends S3BaseCsvDestinationAcceptanceTest {
@@ -13,4 +14,9 @@ public ProtocolVersion getProtocolVersion() {
     return ProtocolVersion.V1;
   }
 
+  @Override
+  protected JsonNode getBaseConfigJson() {
+    return S3DestinationTestUtils.getBaseConfigJsonFilePath();
+  }
+
 }
diff --git a/...ation/java/io/airbyte/integrations/destination/s3/S3CsvGzipDestinationAcceptanceTest.java b/...ation/java/io/airbyte/integrations/destination/s3/S3CsvGzipDestinationAcceptanceTest.java
@@ -4,6 +4,7 @@
 
 package io.airbyte.integrations.destination.s3;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import io.airbyte.integrations.standardtest.destination.ProtocolVersion;
 
 public class S3CsvGzipDestinationAcceptanceTest extends S3BaseCsvGzipDestinationAcceptanceTest {
@@ -13,4 +14,9 @@ public ProtocolVersion getProtocolVersion() {
     return ProtocolVersion.V1;
   }
 
+  @Override
+  protected JsonNode getBaseConfigJson() {
+    return S3DestinationTestUtils.getBaseConfigJsonFilePath();
+  }
+
 }
diff --git a/.../test-integration/java/io/airbyte/integrations/destination/s3/S3DestinationTestUtils.java b/.../test-integration/java/io/airbyte/integrations/destination/s3/S3DestinationTestUtils.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.integrations.destination.s3;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.airbyte.commons.io.IOs;
+import io.airbyte.commons.json.Jsons;
+import java.nio.file.Path;
+
+public class S3DestinationTestUtils {
+
+  private static final String SECRET_PATH = "secrets/s3_dest_min_required_permissions_creds.json";
+
+  public static JsonNode getBaseConfigJsonFilePath() {
+    return Jsons.deserialize(IOs.readFile(Path.of(SECRET_PATH)));
+  }
+
+}
diff --git a/...gration/java/io/airbyte/integrations/destination/s3/S3JsonlDestinationAcceptanceTest.java b/...gration/java/io/airbyte/integrations/destination/s3/S3JsonlDestinationAcceptanceTest.java
@@ -4,6 +4,7 @@
 
 package io.airbyte.integrations.destination.s3;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import io.airbyte.integrations.standardtest.destination.ProtocolVersion;
 
 public class S3JsonlDestinationAcceptanceTest extends S3BaseJsonlDestinationAcceptanceTest {
@@ -13,4 +14,9 @@ public ProtocolVersion getProtocolVersion() {
     return ProtocolVersion.V1;
   }
 
+  @Override
+  protected JsonNode getBaseConfigJson() {
+    return S3DestinationTestUtils.getBaseConfigJsonFilePath();
+  }
+
 }
diff --git a/...ion/java/io/airbyte/integrations/destination/s3/S3JsonlGzipDestinationAcceptanceTest.java b/...ion/java/io/airbyte/integrations/destination/s3/S3JsonlGzipDestinationAcceptanceTest.java
@@ -4,6 +4,7 @@
 
 package io.airbyte.integrations.destination.s3;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import io.airbyte.integrations.standardtest.destination.ProtocolVersion;
 
 public class S3JsonlGzipDestinationAcceptanceTest extends S3BaseJsonlGzipDestinationAcceptanceTest {
@@ -13,4 +14,9 @@ public ProtocolVersion getProtocolVersion() {
     return ProtocolVersion.V1;
   }
 
+  @Override
+  protected JsonNode getBaseConfigJson() {
+    return S3DestinationTestUtils.getBaseConfigJsonFilePath();
+  }
+
 }
diff --git a/...ation/java/io/airbyte/integrations/destination/s3/S3ParquetDestinationAcceptanceTest.java b/...ation/java/io/airbyte/integrations/destination/s3/S3ParquetDestinationAcceptanceTest.java
@@ -4,6 +4,7 @@
 
 package io.airbyte.integrations.destination.s3;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import io.airbyte.integrations.standardtest.destination.ProtocolVersion;
 import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
 
@@ -19,4 +20,9 @@ protected TestDataComparator getTestDataComparator() {
     return new S3AvroParquetTestDataComparator();
   }
 
+  @Override
+  protected JsonNode getBaseConfigJson() {
+    return S3DestinationTestUtils.getBaseConfigJsonFilePath();
+  }
+
 }
diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md
@@ -91,19 +91,28 @@ NOTE: If the S3 cluster is not configured to use TLS, the connection to Amazon S
 
 5. Click `Set up destination`.
 
-In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and "S3 Access Key" are used have access to both the bucket and its contents. Policies to use:
+In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and "S3 Access Key" are used have access to both the bucket and its contents. Minimum required Policies to use:
 ```json
 {
   "Version": "2012-10-17",
     "Statement": [
         {
             "Effect": "Allow",
-            "Action": "s3:*",
+             "Action": [
+                "s3:PutObject",
+                "s3:GetObject",
+                "s3:DeleteObject",
+                "s3:PutObjectAcl",
+                "s3:ListBucket",
+                "s3:ListBucketMultipartUploads",
+                "s3:AbortMultipartUpload",
+                "s3:GetBucketLocation"
+             ],
             "Resource": [
                 "arn:aws:s3:::YOUR_BUCKET_NAME/*",
                 "arn:aws:s3:::YOUR_BUCKET_NAME"
             ]
-            }
+        }
     ]
 }
 ```