Add intelligent tiering storage class

presto-docs/src/main/sphinx/connector/hive.rst

@@ -242,6 +242,10 @@ Property Name                                Description
                                              set this to the AWS region-specific endpoint
                                              (e.g., ``http[s]://<bucket>.s3-<AWS-region>.amazonaws.com``).
 
+``hive.s3.storage-class``                    The S3 storage class to use when writing the data. Currently only
+                                             ``STANDARD`` and ``INTELLIGENT_TIERING`` storage classes are supported.
+                                             Default storage class is ``STANDARD``
+
 ``hive.s3.signer-type``                      Specify a different signer type for S3-compatible storage.
                                              Example: ``S3SignerType`` for v2 signer type
 

presto-hive/src/main/java/com/facebook/presto/hive/s3/HiveS3Config.java

@@ -35,6 +35,7 @@ public class HiveS3Config
     private String s3AwsAccessKey;
     private String s3AwsSecretKey;
     private String s3Endpoint;
+    private PrestoS3StorageClass s3StorageClass = PrestoS3StorageClass.STANDARD;
     private PrestoS3SignerType s3SignerType;
     private boolean s3PathStyleAccess;
     private boolean s3UseInstanceCredentials;
@@ -98,6 +99,20 @@ public HiveS3Config setS3Endpoint(String s3Endpoint)
         return this;
     }
 
+    @NotNull
+    public PrestoS3StorageClass getS3StorageClass()
+    {
+        return s3StorageClass;
+    }
+
+    @Config("hive.s3.storage-class")
+    @ConfigDescription("AWS S3 storage class to use when writing the data")
+    public HiveS3Config setS3StorageClass(PrestoS3StorageClass s3StorageClass)
+    {
+        this.s3StorageClass = s3StorageClass;
+        return this;
+    }
+
     public PrestoS3SignerType getS3SignerType()
     {
         return s3SignerType;

presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3ConfigurationUpdater.java

@@ -27,6 +27,7 @@ public class PrestoS3ConfigurationUpdater
     private final String awsAccessKey;
     private final String awsSecretKey;
     private final String endpoint;
+    private final PrestoS3StorageClass s3StorageClass;
     private final PrestoS3SignerType signerType;
     private final boolean pathStyleAccess;
     private final boolean useInstanceCredentials;
@@ -58,6 +59,7 @@ public PrestoS3ConfigurationUpdater(HiveS3Config config)
         this.awsAccessKey = config.getS3AwsAccessKey();
         this.awsSecretKey = config.getS3AwsSecretKey();
         this.endpoint = config.getS3Endpoint();
+        this.s3StorageClass = config.getS3StorageClass();
         this.signerType = config.getS3SignerType();
         this.pathStyleAccess = config.isS3PathStyleAccess();
         this.useInstanceCredentials = config.isS3UseInstanceCredentials();
@@ -101,6 +103,7 @@ public void updateConfiguration(Configuration config)
         if (endpoint != null) {
             config.set(S3_ENDPOINT, endpoint);
         }
+        config.set(S3_STORAGE_CLASS, s3StorageClass.name());
         if (signerType != null) {
             config.set(S3_SIGNER_TYPE, signerType.name());
         }

presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3FileSystem.java

@@ -46,6 +46,7 @@
 import com.amazonaws.services.s3.model.S3ObjectInputStream;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
+import com.amazonaws.services.s3.model.StorageClass;
 import com.amazonaws.services.s3.transfer.Transfer;
 import com.amazonaws.services.s3.transfer.TransferManager;
 import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
@@ -127,6 +128,7 @@
 import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_SSE_TYPE;
 import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_SSL_ENABLED;
 import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_STAGING_DIRECTORY;
+import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_STORAGE_CLASS;
 import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_USER_AGENT_PREFIX;
 import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_USER_AGENT_SUFFIX;
 import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_USE_INSTANCE_CREDENTIALS;
@@ -187,6 +189,7 @@ public class PrestoS3FileSystem
     private long multiPartUploadMinPartSize;
     private PrestoS3AclType s3AclType;
     private boolean skipGlacierObjects;
+    private PrestoS3StorageClass s3StorageClass;
 
     @Override
     public void initialize(URI uri, Configuration conf)
@@ -227,6 +230,7 @@ public void initialize(URI uri, Configuration conf)
         this.s3AclType = PrestoS3AclType.valueOf(conf.get(S3_ACL_TYPE, defaults.getS3AclType().name()));
         String userAgentPrefix = conf.get(S3_USER_AGENT_PREFIX, defaults.getS3UserAgentPrefix());
         this.skipGlacierObjects = conf.getBoolean(S3_SKIP_GLACIER_OBJECTS, defaults.isSkipGlacierObjects());
+        this.s3StorageClass = conf.getEnum(S3_STORAGE_CLASS, defaults.getS3StorageClass());
 
         ClientConfiguration configuration = new ClientConfiguration()
                 .withMaxErrorRetry(maxErrorRetries)
@@ -405,7 +409,17 @@ public FSDataOutputStream create(Path path, FsPermission permission, boolean ove
 
         String key = keyFromPath(qualifiedPath(path));
         return new FSDataOutputStream(
-                new PrestoS3OutputStream(s3, getBucketName(uri), key, tempFile, sseEnabled, sseType, sseKmsKeyId, multiPartUploadMinFileSize, multiPartUploadMinPartSize, s3AclType),
+                new PrestoS3OutputStream(s3,
+                        getBucketName(uri),
+                        key,
+                        tempFile,
+                        sseEnabled,
+                        sseType,
+                        sseKmsKeyId,
+                        multiPartUploadMinFileSize,
+                        multiPartUploadMinPartSize,
+                        s3AclType,
+                        s3StorageClass),
                 statistics);
     }
 
@@ -1159,6 +1173,7 @@ private static class PrestoS3OutputStream
         private final PrestoS3SseType sseType;
         private final String sseKmsKeyId;
         private final CannedAccessControlList aclType;
+        private final StorageClass s3StorageClass;
 
         private boolean closed;
 
@@ -1172,7 +1187,8 @@ public PrestoS3OutputStream(
                 String sseKmsKeyId,
                 long multiPartUploadMinFileSize,
                 long multiPartUploadMinPartSize,
-                PrestoS3AclType aclType)
+                PrestoS3AclType aclType,
+                PrestoS3StorageClass s3StorageClass)
                 throws IOException
         {
             super(new BufferedOutputStream(Files.newOutputStream(requireNonNull(tempFile, "tempFile is null").toPath())));
@@ -1183,13 +1199,15 @@ public PrestoS3OutputStream(
                     .withMultipartUploadThreshold(multiPartUploadMinFileSize).build();
 
             requireNonNull(aclType, "aclType is null");
+            requireNonNull(s3StorageClass, "s3StorageClass is null");
             this.aclType = aclType.getCannedACL();
             this.host = requireNonNull(host, "host is null");
             this.key = requireNonNull(key, "key is null");
             this.tempFile = tempFile;
             this.sseEnabled = sseEnabled;
             this.sseType = requireNonNull(sseType, "sseType is null");
             this.sseKmsKeyId = sseKmsKeyId;
+            this.s3StorageClass = s3StorageClass.getS3StorageClass();
 
             log.debug("OutputStream for key '%s' using file: %s", key, tempFile);
         }
@@ -1241,6 +1259,7 @@ private void uploadObject()
                             break;
                     }
                 }
+                request.withStorageClass(s3StorageClass);
 
                 request.withCannedAcl(aclType);
 

presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3StorageClass.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.hive.s3;
+
+import com.amazonaws.services.s3.model.StorageClass;
+
+import static java.util.Objects.requireNonNull;
+
+public enum PrestoS3StorageClass {
+    STANDARD(StorageClass.Standard),
+    INTELLIGENT_TIERING(StorageClass.IntelligentTiering);
+
+    private StorageClass s3StorageClass;
+
+    PrestoS3StorageClass(StorageClass s3StorageClass)
+    {
+        this.s3StorageClass = requireNonNull(s3StorageClass, "s3StorageClass is null");
+    }
+
+    public StorageClass getS3StorageClass()
+    {
+        return s3StorageClass;
+    }
+}

presto-hive/src/main/java/com/facebook/presto/hive/s3/S3ConfigurationUpdater.java

@@ -47,6 +47,7 @@ public interface S3ConfigurationUpdater
     String S3_ACCESS_KEY = "presto.s3.access-key";
     String S3_ACL_TYPE = "presto.s3.upload-acl-type";
     String S3_SKIP_GLACIER_OBJECTS = "presto.s3.skip-glacier-objects";
+    String S3_STORAGE_CLASS = "presto.s3.storage-class";
 
     void updateConfiguration(Configuration config);
 }

presto-hive/src/test/java/com/facebook/presto/hive/s3/TestHiveS3Config.java

@@ -41,6 +41,7 @@ public void testDefaults()
                 .setS3PathStyleAccess(false)
                 .setS3UseInstanceCredentials(false)
                 .setS3IamRole(null)
+                .setS3StorageClass(PrestoS3StorageClass.STANDARD)
                 .setS3IamRoleSessionName("presto-session")
                 .setS3SslEnabled(true)
                 .setS3SseEnabled(false)
@@ -75,6 +76,7 @@ public void testExplicitPropertyMappings()
                 .put("hive.s3.path-style-access", "true")
                 .put("hive.s3.use-instance-credentials", "true")
                 .put("hive.s3.iam-role", "roleArn")
+                .put("hive.s3.storage-class", "INTELLIGENT_TIERING")
                 .put("hive.s3.iam-role-session-name", "roleSessionName")
                 .put("hive.s3.ssl.enabled", "false")
                 .put("hive.s3.sse.enabled", "true")
@@ -106,6 +108,7 @@ public void testExplicitPropertyMappings()
                 .setS3PathStyleAccess(true)
                 .setS3UseInstanceCredentials(true)
                 .setS3IamRole("roleArn")
+                .setS3StorageClass(PrestoS3StorageClass.INTELLIGENT_TIERING)
                 .setS3IamRoleSessionName("roleSessionName")
                 .setS3SslEnabled(false)
                 .setS3SseEnabled(true)