Added region property for billing buckets to reference properly if v4…

… auth
Teevity · Dec 4, 2015 · 03073a3 · 03073a3
1 parent 5c76154
commit 03073a3
Show file tree

Hide file tree

Showing 6 changed files with 26 additions and 8 deletions.
diff --git a/grails-app/conf/BootStrap.groovy b/grails-app/conf/BootStrap.groovy
@@ -145,6 +145,7 @@ class BootStrap {
 
                 properties.setProperty(IceOptions.LOCAL_DIR, prop.getProperty("ice.processor.localDir"));
                 properties.setProperty(IceOptions.BILLING_S3_BUCKET_NAME, prop.getProperty(IceOptions.BILLING_S3_BUCKET_NAME));
+                properties.setProperty(IceOptions.BILLING_S3_BUCKET_REGION, prop.getProperty(IceOptions.BILLING_S3_BUCKET_REGION));
                 properties.setProperty(IceOptions.BILLING_S3_BUCKET_PREFIX, prop.getProperty(IceOptions.BILLING_S3_BUCKET_PREFIX, ""));
                 properties.setProperty(IceOptions.BILLING_PAYER_ACCOUNT_ID, prop.getProperty(IceOptions.BILLING_PAYER_ACCOUNT_ID, ""));
                 properties.setProperty(IceOptions.BILLING_ACCESS_ROLENAME, prop.getProperty(IceOptions.BILLING_ACCESS_ROLENAME, ""));

diff --git a/src/java/com/netflix/ice/common/AwsUtils.java b/src/java/com/netflix/ice/common/AwsUtils.java
@@ -258,8 +258,8 @@ public static long getLastModified(String bucketName, String fileKey) {
         }
     }
 
-    public static boolean downloadFileIfChangedSince(String bucketName, String bucketFilePrefix, File file, long milles, String accountId,
-                                                     String assumeRole, String externalId) {
+    public static boolean downloadFileIfChangedSince(String bucketName, String bucketFileRegion, String bucketFilePrefix, File file,
+                                                     long milles, String accountId, String assumeRole, String externalId) {
         AmazonS3Client s3Client = AwsUtils.s3Client;
 
         try {
@@ -272,6 +272,10 @@ public static boolean downloadFileIfChangedSince(String bucketName, String bucke
                                 clientConfig);
             }
 
+            if(bucketFileRegion != null && !bucketFileRegion.isEmpty()) {
+                s3Client.setEndpoint("s3-" + bucketFileRegion + ".amazonaws.com");
+            }
+
             ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, bucketFilePrefix + file.getName());
             boolean download = !file.exists() || metadata.getLastModified().getTime() > milles;
 

diff --git a/src/java/com/netflix/ice/common/IceOptions.java b/src/java/com/netflix/ice/common/IceOptions.java
@@ -51,6 +51,12 @@ public class IceOptions {
      */
     public static final String BILLING_S3_BUCKET_NAME = "ice.billing_s3bucketname";
 
+    /**
+     * Region for billing s3 bucket. It should be specified for buckets using v4 validation ",".
+     * It must be specified in Config.
+     */
+    public static final String BILLING_S3_BUCKET_REGION = "ice.billing_s3bucketregion";
+
     /**
      * Prefix of billing files in billing s3 bucket. For multiple payer accounts, multiple bucket prefixes can be specified delimited by comma ",".
      * It must be specified in Config.

diff --git a/src/java/com/netflix/ice/processor/BillingFileProcessor.java b/src/java/com/netflix/ice/processor/BillingFileProcessor.java
@@ -77,6 +77,7 @@ protected void poll() throws Exception {
         // list the tar.gz file in billing file folder
         for (int i = 0; i < config.billingS3BucketNames.length; i++) {
             String billingS3BucketName = config.billingS3BucketNames[i];
+            String billingS3BucketRegion = config.billingS3BucketRegions.length > i ? config.billingS3BucketRegions[i] : "";
             String billingS3BucketPrefix = config.billingS3BucketPrefixes.length > i ? config.billingS3BucketPrefixes[i] : "";
             String accountId = config.billingAccountIds.length > i ? config.billingAccountIds[i] : "";
             String billingAccessRoleName = config.billingAccessRoleNames.length > i ? config.billingAccessRoleNames[i] : "";
@@ -128,7 +129,7 @@ protected void poll() throws Exception {
                     list = Lists.newArrayList();
                     filesToProcess.put(key, list);
                 }
-                list.add(new BillingFile(filesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix));
+                list.add(new BillingFile(filesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix, billingS3BucketRegion));
             }
 
             for (DateTime key: monitorFilesToProcessInOneBucket.keySet()) {
@@ -137,7 +138,7 @@ protected void poll() throws Exception {
                     list = Lists.newArrayList();
                     monitorFilesToProcess.put(key, list);
                 }
-                list.add(new BillingFile(monitorFilesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix));
+                list.add(new BillingFile(monitorFilesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix, billingS3BucketRegion));
             }
         }
 
@@ -172,7 +173,7 @@ protected void poll() throws Exception {
 
                 File file = new File(config.localDir, fileKey.substring(billingFile.prefix.length()));
                 logger.info("trying to download " + fileKey + "...");
-                boolean downloaded = AwsUtils.downloadFileIfChangedSince(objectSummary.getBucketName(), billingFile.prefix, file, lastProcessed,
+                boolean downloaded = AwsUtils.downloadFileIfChangedSince(objectSummary.getBucketName(), billingFile.region, billingFile.prefix, file, lastProcessed,
                         billingFile.accountId, billingFile.accessRoleName, billingFile.externalId);
                 if (downloaded)
                     logger.info("downloaded " + fileKey);
@@ -197,8 +198,8 @@ protected void poll() throws Exception {
                         logger.info("processing " + monitorFileKey + "...");
                         File monitorFile = new File(config.localDir, monitorFileKey.substring(monitorFileKey.lastIndexOf("/") + 1));
                         logger.info("trying to download " + monitorFileKey + "...");
-                        boolean downloaded = AwsUtils.downloadFileIfChangedSince(monitorObjectSummary.getBucketName(), monitorBillingFile.prefix, monitorFile, lastProcessed,
-                                monitorBillingFile.accountId, monitorBillingFile.accessRoleName, monitorBillingFile.externalId);
+                        boolean downloaded = AwsUtils.downloadFileIfChangedSince(monitorObjectSummary.getBucketName(), monitorBillingFile.region, monitorBillingFile.prefix,
+                                monitorFile, lastProcessed, monitorBillingFile.accountId, monitorBillingFile.accessRoleName, monitorBillingFile.externalId);
                         if (downloaded)
                             logger.info("downloaded " + monitorFile);
                         else
@@ -741,13 +742,15 @@ endMilli < lastAlertMillis() + AwsUtils.hourMillis * 24)
 
     private class BillingFile {
         final S3ObjectSummary s3ObjectSummary;
+        final String region;
         final String accountId;
         final String accessRoleName;
         final String externalId;
         final String prefix;
 
-        BillingFile(S3ObjectSummary s3ObjectSummary, String accountId, String accessRoleName, String externalId, String prefix) {
+        BillingFile(S3ObjectSummary s3ObjectSummary, String accountId, String accessRoleName, String externalId, String prefix, String region) {
             this.s3ObjectSummary = s3ObjectSummary;
+            this.region = region;
             this.accountId = accountId;
             this.accessRoleName = accessRoleName;
             this.externalId = externalId;

diff --git a/src/java/com/netflix/ice/processor/ProcessorConfig.java b/src/java/com/netflix/ice/processor/ProcessorConfig.java
@@ -32,6 +32,7 @@ public class ProcessorConfig extends Config {
 
     public final String[] billingAccountIds;
     public final String[] billingS3BucketNames;
+    public final String[] billingS3BucketRegions;
     public final String[] billingS3BucketPrefixes;
     public final String[] billingAccessRoleNames;
     public final String[] billingAccessExternalIds;
@@ -80,6 +81,7 @@ public ProcessorConfig(
             this.costPerMonitorMetricPerHour = 0;
 
         billingS3BucketNames = properties.getProperty(IceOptions.BILLING_S3_BUCKET_NAME).split(",");
+        billingS3BucketRegions = properties.getProperty(IceOptions.BILLING_S3_BUCKET_REGION).split(",");
         billingS3BucketPrefixes = properties.getProperty(IceOptions.BILLING_S3_BUCKET_PREFIX, "").split(",");
         billingAccountIds = properties.getProperty(IceOptions.BILLING_PAYER_ACCOUNT_ID, "").split(",");
         billingAccessRoleNames = properties.getProperty(IceOptions.BILLING_ACCESS_ROLENAME, "").split(",");

diff --git a/src/java/sample.properties b/src/java/sample.properties
@@ -31,6 +31,8 @@ ice.ondemandCostAlertEmails=
 # modify the following 5 properties according to your billing files configuration. if you have multiple payer accounts, you will need to specify multiple values for each property.
 # s3 bucket name where the billing files are. multiple bucket names are delimited by ",". Ice must have read access to billing s3 bucket.
 ice.billing_s3bucketname=billing_s3bucketname1,billing_s3bucketname2
+# location for the billing bucket.  It should be specified for buckets using v4 validation
+ice.billing_s3bucketregion=eu-west-1,eu-central-1
 # prefix of the billing files. multiple prefixes are delimited by ","
 ice.billing_s3bucketprefix=,
 # specify your payer account id here if across-accounts IAM role access is used. multiple account ids are delimited by ",". "ice.billing_payerAccountId=,222222222222" means assumed role access is only used for the second bucket.