diff --git a/grails-app/conf/BootStrap.groovy b/grails-app/conf/BootStrap.groovy index 44e978bf..19d1986f 100644 --- a/grails-app/conf/BootStrap.groovy +++ b/grails-app/conf/BootStrap.groovy @@ -145,6 +145,7 @@ class BootStrap { properties.setProperty(IceOptions.LOCAL_DIR, prop.getProperty("ice.processor.localDir")); properties.setProperty(IceOptions.BILLING_S3_BUCKET_NAME, prop.getProperty(IceOptions.BILLING_S3_BUCKET_NAME)); + properties.setProperty(IceOptions.BILLING_S3_BUCKET_REGION, prop.getProperty(IceOptions.BILLING_S3_BUCKET_REGION)); properties.setProperty(IceOptions.BILLING_S3_BUCKET_PREFIX, prop.getProperty(IceOptions.BILLING_S3_BUCKET_PREFIX, "")); properties.setProperty(IceOptions.BILLING_PAYER_ACCOUNT_ID, prop.getProperty(IceOptions.BILLING_PAYER_ACCOUNT_ID, "")); properties.setProperty(IceOptions.BILLING_ACCESS_ROLENAME, prop.getProperty(IceOptions.BILLING_ACCESS_ROLENAME, "")); diff --git a/src/java/com/netflix/ice/common/AwsUtils.java b/src/java/com/netflix/ice/common/AwsUtils.java index d4562315..2efabd7c 100755 --- a/src/java/com/netflix/ice/common/AwsUtils.java +++ b/src/java/com/netflix/ice/common/AwsUtils.java @@ -258,8 +258,8 @@ public static long getLastModified(String bucketName, String fileKey) { } } - public static boolean downloadFileIfChangedSince(String bucketName, String bucketFilePrefix, File file, long milles, String accountId, - String assumeRole, String externalId) { + public static boolean downloadFileIfChangedSince(String bucketName, String bucketFileRegion, String bucketFilePrefix, File file, + long milles, String accountId, String assumeRole, String externalId) { AmazonS3Client s3Client = AwsUtils.s3Client; try { @@ -272,6 +272,10 @@ public static boolean downloadFileIfChangedSince(String bucketName, String bucke clientConfig); } + if(bucketFileRegion != null && !bucketFileRegion.isEmpty()) { + s3Client.setEndpoint("s3-" + bucketFileRegion + ".amazonaws.com"); + } + ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, bucketFilePrefix + file.getName()); boolean download = !file.exists() || metadata.getLastModified().getTime() > milles; diff --git a/src/java/com/netflix/ice/common/IceOptions.java b/src/java/com/netflix/ice/common/IceOptions.java index e78077c1..28c9ed99 100644 --- a/src/java/com/netflix/ice/common/IceOptions.java +++ b/src/java/com/netflix/ice/common/IceOptions.java @@ -51,6 +51,12 @@ public class IceOptions { */ public static final String BILLING_S3_BUCKET_NAME = "ice.billing_s3bucketname"; + /** + * Region for billing s3 bucket. It should be specified for buckets using v4 validation ",". + * It must be specified in Config. + */ + public static final String BILLING_S3_BUCKET_REGION = "ice.billing_s3bucketregion"; + /** * Prefix of billing files in billing s3 bucket. For multiple payer accounts, multiple bucket prefixes can be specified delimited by comma ",". * It must be specified in Config. diff --git a/src/java/com/netflix/ice/processor/BillingFileProcessor.java b/src/java/com/netflix/ice/processor/BillingFileProcessor.java index d8bd990c..ac907265 100644 --- a/src/java/com/netflix/ice/processor/BillingFileProcessor.java +++ b/src/java/com/netflix/ice/processor/BillingFileProcessor.java @@ -77,6 +77,7 @@ protected void poll() throws Exception { // list the tar.gz file in billing file folder for (int i = 0; i < config.billingS3BucketNames.length; i++) { String billingS3BucketName = config.billingS3BucketNames[i]; + String billingS3BucketRegion = config.billingS3BucketRegions.length > i ? config.billingS3BucketRegions[i] : ""; String billingS3BucketPrefix = config.billingS3BucketPrefixes.length > i ? config.billingS3BucketPrefixes[i] : ""; String accountId = config.billingAccountIds.length > i ? config.billingAccountIds[i] : ""; String billingAccessRoleName = config.billingAccessRoleNames.length > i ? config.billingAccessRoleNames[i] : ""; @@ -128,7 +129,7 @@ protected void poll() throws Exception { list = Lists.newArrayList(); filesToProcess.put(key, list); } - list.add(new BillingFile(filesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix)); + list.add(new BillingFile(filesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix, billingS3BucketRegion)); } for (DateTime key: monitorFilesToProcessInOneBucket.keySet()) { @@ -137,7 +138,7 @@ protected void poll() throws Exception { list = Lists.newArrayList(); monitorFilesToProcess.put(key, list); } - list.add(new BillingFile(monitorFilesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix)); + list.add(new BillingFile(monitorFilesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix, billingS3BucketRegion)); } } @@ -172,7 +173,7 @@ protected void poll() throws Exception { File file = new File(config.localDir, fileKey.substring(billingFile.prefix.length())); logger.info("trying to download " + fileKey + "..."); - boolean downloaded = AwsUtils.downloadFileIfChangedSince(objectSummary.getBucketName(), billingFile.prefix, file, lastProcessed, + boolean downloaded = AwsUtils.downloadFileIfChangedSince(objectSummary.getBucketName(), billingFile.region, billingFile.prefix, file, lastProcessed, billingFile.accountId, billingFile.accessRoleName, billingFile.externalId); if (downloaded) logger.info("downloaded " + fileKey); @@ -197,8 +198,8 @@ protected void poll() throws Exception { logger.info("processing " + monitorFileKey + "..."); File monitorFile = new File(config.localDir, monitorFileKey.substring(monitorFileKey.lastIndexOf("/") + 1)); logger.info("trying to download " + monitorFileKey + "..."); - boolean downloaded = AwsUtils.downloadFileIfChangedSince(monitorObjectSummary.getBucketName(), monitorBillingFile.prefix, monitorFile, lastProcessed, - monitorBillingFile.accountId, monitorBillingFile.accessRoleName, monitorBillingFile.externalId); + boolean downloaded = AwsUtils.downloadFileIfChangedSince(monitorObjectSummary.getBucketName(), monitorBillingFile.region, monitorBillingFile.prefix, + monitorFile, lastProcessed, monitorBillingFile.accountId, monitorBillingFile.accessRoleName, monitorBillingFile.externalId); if (downloaded) logger.info("downloaded " + monitorFile); else @@ -741,13 +742,15 @@ endMilli < lastAlertMillis() + AwsUtils.hourMillis * 24) private class BillingFile { final S3ObjectSummary s3ObjectSummary; + final String region; final String accountId; final String accessRoleName; final String externalId; final String prefix; - BillingFile(S3ObjectSummary s3ObjectSummary, String accountId, String accessRoleName, String externalId, String prefix) { + BillingFile(S3ObjectSummary s3ObjectSummary, String accountId, String accessRoleName, String externalId, String prefix, String region) { this.s3ObjectSummary = s3ObjectSummary; + this.region = region; this.accountId = accountId; this.accessRoleName = accessRoleName; this.externalId = externalId; diff --git a/src/java/com/netflix/ice/processor/ProcessorConfig.java b/src/java/com/netflix/ice/processor/ProcessorConfig.java index f4906c89..3c740aa8 100644 --- a/src/java/com/netflix/ice/processor/ProcessorConfig.java +++ b/src/java/com/netflix/ice/processor/ProcessorConfig.java @@ -32,6 +32,7 @@ public class ProcessorConfig extends Config { public final String[] billingAccountIds; public final String[] billingS3BucketNames; + public final String[] billingS3BucketRegions; public final String[] billingS3BucketPrefixes; public final String[] billingAccessRoleNames; public final String[] billingAccessExternalIds; @@ -80,6 +81,7 @@ public ProcessorConfig( this.costPerMonitorMetricPerHour = 0; billingS3BucketNames = properties.getProperty(IceOptions.BILLING_S3_BUCKET_NAME).split(","); + billingS3BucketRegions = properties.getProperty(IceOptions.BILLING_S3_BUCKET_REGION).split(","); billingS3BucketPrefixes = properties.getProperty(IceOptions.BILLING_S3_BUCKET_PREFIX, "").split(","); billingAccountIds = properties.getProperty(IceOptions.BILLING_PAYER_ACCOUNT_ID, "").split(","); billingAccessRoleNames = properties.getProperty(IceOptions.BILLING_ACCESS_ROLENAME, "").split(","); diff --git a/src/java/sample.properties b/src/java/sample.properties index 09c96bd7..895e0d8d 100644 --- a/src/java/sample.properties +++ b/src/java/sample.properties @@ -31,6 +31,8 @@ ice.ondemandCostAlertEmails= # modify the following 5 properties according to your billing files configuration. if you have multiple payer accounts, you will need to specify multiple values for each property. # s3 bucket name where the billing files are. multiple bucket names are delimited by ",". Ice must have read access to billing s3 bucket. ice.billing_s3bucketname=billing_s3bucketname1,billing_s3bucketname2 +# location for the billing bucket. It should be specified for buckets using v4 validation +ice.billing_s3bucketregion=eu-west-1,eu-central-1 # prefix of the billing files. multiple prefixes are delimited by "," ice.billing_s3bucketprefix=, # specify your payer account id here if across-accounts IAM role access is used. multiple account ids are delimited by ",". "ice.billing_payerAccountId=,222222222222" means assumed role access is only used for the second bucket.