Merge branch 'integration'

* integration:
  Ignore the Region-Wide RIs when generating the reservation prices files
  Updated transitive dependencies for aws-java-sdk
  Update SDK to version 1.1.136 + Add org.json as a separate library since it has been removed from the AWS SDK
  Added more temporary build folders in the gitignore (+ OSX and FUSE)
  added initial .gitignore #3
  Added eu-west-2 and ca-central-1 + Fixed ap-south-1 which was not put in regionsByName
  Reverted changed on ap-southeast-* short names + Fixed ap-south-1 short name
  Update Region.java
  Add missing regions and zones
  Adding ap-southeast-2c
  Update aws-sdk to latest version
  Pin highcharts version. Closes #210
  Added region property for billing buckets to reference properly if v4 auth
  Added commons-io dependency

.gitignore

@@ -0,0 +1,54 @@
+# .gitignore for Grails 1.2 and 1.3
+
+# web application files
+/web-app/WEB-INF
+
+# IDE support files
+/.classpath
+/.launch
+/.project
+/.settings
+/*.launch
+/*.tmproj
+/ivy*
+/eclipse
+
+# default HSQL database files for production mode
+/prodDb.*
+
+# general HSQL database files
+*Db.properties
+*Db.script
+
+# logs
+/stacktrace.log
+/test/reports
+/logs
+
+# project release file
+/*.war
+
+# plugin release file
+/*.zip
+
+# older plugin install locations
+/plugins
+/web-app/plugins
+/web-app/WEB-INF/classes
+
+# "temporary" build files
+/target
+/target-eclipse
+/work
+
+# other
+*.iws
+
+#project specific:
+/src/java/ice.properties
+
+# OSX
+.DS_Store
+
+# FUSE files
+.fuse_hidden*

grails-app/conf/BootStrap.groovy

@@ -145,6 +145,7 @@ class BootStrap {
 
                 properties.setProperty(IceOptions.LOCAL_DIR, prop.getProperty("ice.processor.localDir"));
                 properties.setProperty(IceOptions.BILLING_S3_BUCKET_NAME, prop.getProperty(IceOptions.BILLING_S3_BUCKET_NAME));
+                properties.setProperty(IceOptions.BILLING_S3_BUCKET_REGION, prop.getProperty(IceOptions.BILLING_S3_BUCKET_REGION));
                 properties.setProperty(IceOptions.BILLING_S3_BUCKET_PREFIX, prop.getProperty(IceOptions.BILLING_S3_BUCKET_PREFIX, ""));
                 properties.setProperty(IceOptions.BILLING_PAYER_ACCOUNT_ID, prop.getProperty(IceOptions.BILLING_PAYER_ACCOUNT_ID, ""));
                 properties.setProperty(IceOptions.BILLING_ACCESS_ROLENAME, prop.getProperty(IceOptions.BILLING_ACCESS_ROLENAME, ""));

grails-app/conf/BuildConfig.groovy

@@ -66,12 +66,12 @@ grails.project.dependency.resolution = {
 
         compile(
                 // Amazon Web Services programmatic interface
-                'com.amazonaws:aws-java-sdk:1.9.12',
+                'com.amazonaws:aws-java-sdk:1.11.136',
                 // Transitive dependencies of aws-java-sdk, but also used directly.
                 // It would be great if we could upgrade httpcore and httpclient, but we can't until the AWS Java SDK
                 // upgrades its dependencies. If we simply upgrade these, then some Amazon calls fail.
-                'org.apache.httpcomponents:httpcore:4.2',
-                'org.apache.httpcomponents:httpclient:4.2',
+                'org.apache.httpcomponents:httpcore:4.4.4',
+                'org.apache.httpcomponents:httpclient:4.5.2',
 
                 // Explicitly including aws-java-sdk transitive dependencies
                 'org.codehaus.jackson:jackson-core-asl:1.8.9',
@@ -85,6 +85,9 @@ grails.project.dependency.resolution = {
 
                 // Better Zip Support
                 'org.apache.commons:commons-compress:1.8',
+
+                // Better IO Support
+                'commons-io:commons-io:2.4',
 
                 // Easier Java from Joshua Bloch and Google
                 'com.google.guava:guava:14.0',
@@ -101,7 +104,10 @@ grails.project.dependency.resolution = {
                 'org.codehaus.woodstox:wstx-asl:3.2.9',
                 'jfree:jfreechart:1.0.13',
                 'org.json:json:20090211',
-                'org.mapdb:mapdb:0.9.1'
+                'org.mapdb:mapdb:0.9.1',
+
+                // Since the AWS SDK has removed its package "com.amazonaws.util.json", we need to include it as a separate library 
+                'org.json:json:20090211'
 
         ) { // Exclude superfluous and dangerous transitive dependencies
             excludes(

src/java/com/netflix/ice/basic/BasicReservationService.java

@@ -155,6 +155,11 @@ private void pollAPI() throws Exception {
                 for (ReservedInstancesOffering offer: offers.getReservedInstancesOfferings()) {
                     if (offer.getProductDescription().indexOf("Amazon VPC") >= 0)
                         continue;
+
+                    // Ignore Region-Wide RIs
+                    if (offer.getAvailabilityZone() == null)
+                        continue;
+
                     ReservationUtilization utilization = ReservationUtilization.get(offer.getOfferingType());
                     Ec2InstanceReservationPrice.ReservationPeriod term = offer.getDuration() / 24 / 3600 > 366 ?
                             Ec2InstanceReservationPrice.ReservationPeriod.threeyear : Ec2InstanceReservationPrice.ReservationPeriod.oneyear;

src/java/com/netflix/ice/basic/EddaResourceService.java

@@ -10,12 +10,12 @@
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.amazonaws.util.json.JSONArray;
-import com.amazonaws.util.json.JSONException;
-import com.amazonaws.util.json.JSONObject;
 import com.google.common.collect.Lists;
 import com.netflix.ice.common.ResourceService;
 import com.netflix.ice.tag.Account;

src/java/com/netflix/ice/common/AwsUtils.java

@@ -258,8 +258,8 @@ public static long getLastModified(String bucketName, String fileKey) {
         }
     }
 
-    public static boolean downloadFileIfChangedSince(String bucketName, String bucketFilePrefix, File file, long milles, String accountId,
-                                                     String assumeRole, String externalId) {
+    public static boolean downloadFileIfChangedSince(String bucketName, String bucketFileRegion, String bucketFilePrefix, File file,
+                                                     long milles, String accountId, String assumeRole, String externalId) {
         AmazonS3Client s3Client = AwsUtils.s3Client;
 
         try {
@@ -272,6 +272,10 @@ public static boolean downloadFileIfChangedSince(String bucketName, String bucke
                                 clientConfig);
             }
 
+            if(bucketFileRegion != null && !bucketFileRegion.isEmpty()) {
+                s3Client.setEndpoint("s3-" + bucketFileRegion + ".amazonaws.com");
+            }
+
             ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, bucketFilePrefix + file.getName());
             boolean download = !file.exists() || metadata.getLastModified().getTime() > milles;
 

src/java/com/netflix/ice/common/IceOptions.java

@@ -51,6 +51,12 @@ public class IceOptions {
      */
     public static final String BILLING_S3_BUCKET_NAME = "ice.billing_s3bucketname";
 
+    /**
+     * Region for billing s3 bucket. It should be specified for buckets using v4 validation ",".
+     * It must be specified in Config.
+     */
+    public static final String BILLING_S3_BUCKET_REGION = "ice.billing_s3bucketregion";
+
     /**
      * Prefix of billing files in billing s3 bucket. For multiple payer accounts, multiple bucket prefixes can be specified delimited by comma ",".
      * It must be specified in Config.

src/java/com/netflix/ice/processor/BillingFileProcessor.java

@@ -77,6 +77,7 @@ protected void poll() throws Exception {
         // list the tar.gz file in billing file folder
         for (int i = 0; i < config.billingS3BucketNames.length; i++) {
             String billingS3BucketName = config.billingS3BucketNames[i];
+            String billingS3BucketRegion = config.billingS3BucketRegions.length > i ? config.billingS3BucketRegions[i] : "";
             String billingS3BucketPrefix = config.billingS3BucketPrefixes.length > i ? config.billingS3BucketPrefixes[i] : "";
             String accountId = config.billingAccountIds.length > i ? config.billingAccountIds[i] : "";
             String billingAccessRoleName = config.billingAccessRoleNames.length > i ? config.billingAccessRoleNames[i] : "";
@@ -128,7 +129,7 @@ protected void poll() throws Exception {
                     list = Lists.newArrayList();
                     filesToProcess.put(key, list);
                 }
-                list.add(new BillingFile(filesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix));
+                list.add(new BillingFile(filesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix, billingS3BucketRegion));
             }
 
             for (DateTime key: monitorFilesToProcessInOneBucket.keySet()) {
@@ -137,7 +138,7 @@ protected void poll() throws Exception {
                     list = Lists.newArrayList();
                     monitorFilesToProcess.put(key, list);
                 }
-                list.add(new BillingFile(monitorFilesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix));
+                list.add(new BillingFile(monitorFilesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix, billingS3BucketRegion));
             }
         }
 
@@ -172,7 +173,7 @@ protected void poll() throws Exception {
 
                 File file = new File(config.localDir, fileKey.substring(billingFile.prefix.length()));
                 logger.info("trying to download " + fileKey + "...");
-                boolean downloaded = AwsUtils.downloadFileIfChangedSince(objectSummary.getBucketName(), billingFile.prefix, file, lastProcessed,
+                boolean downloaded = AwsUtils.downloadFileIfChangedSince(objectSummary.getBucketName(), billingFile.region, billingFile.prefix, file, lastProcessed,
                         billingFile.accountId, billingFile.accessRoleName, billingFile.externalId);
                 if (downloaded)
                     logger.info("downloaded " + fileKey);
@@ -197,8 +198,8 @@ protected void poll() throws Exception {
                         logger.info("processing " + monitorFileKey + "...");
                         File monitorFile = new File(config.localDir, monitorFileKey.substring(monitorFileKey.lastIndexOf("/") + 1));
                         logger.info("trying to download " + monitorFileKey + "...");
-                        boolean downloaded = AwsUtils.downloadFileIfChangedSince(monitorObjectSummary.getBucketName(), monitorBillingFile.prefix, monitorFile, lastProcessed,
-                                monitorBillingFile.accountId, monitorBillingFile.accessRoleName, monitorBillingFile.externalId);
+                        boolean downloaded = AwsUtils.downloadFileIfChangedSince(monitorObjectSummary.getBucketName(), monitorBillingFile.region, monitorBillingFile.prefix,
+                                monitorFile, lastProcessed, monitorBillingFile.accountId, monitorBillingFile.accessRoleName, monitorBillingFile.externalId);
                         if (downloaded)
                             logger.info("downloaded " + monitorFile);
                         else
@@ -741,13 +742,15 @@ endMilli < lastAlertMillis() + AwsUtils.hourMillis * 24)
 
     private class BillingFile {
         final S3ObjectSummary s3ObjectSummary;
+        final String region;
         final String accountId;
         final String accessRoleName;
         final String externalId;
         final String prefix;
 
-        BillingFile(S3ObjectSummary s3ObjectSummary, String accountId, String accessRoleName, String externalId, String prefix) {
+        BillingFile(S3ObjectSummary s3ObjectSummary, String accountId, String accessRoleName, String externalId, String prefix, String region) {
             this.s3ObjectSummary = s3ObjectSummary;
+            this.region = region;
             this.accountId = accountId;
             this.accessRoleName = accessRoleName;
             this.externalId = externalId;

src/java/com/netflix/ice/processor/ProcessorConfig.java

@@ -32,6 +32,7 @@ public class ProcessorConfig extends Config {
 
     public final String[] billingAccountIds;
     public final String[] billingS3BucketNames;
+    public final String[] billingS3BucketRegions;
     public final String[] billingS3BucketPrefixes;
     public final String[] billingAccessRoleNames;
     public final String[] billingAccessExternalIds;
@@ -80,6 +81,7 @@ public ProcessorConfig(
             this.costPerMonitorMetricPerHour = 0;
 
         billingS3BucketNames = properties.getProperty(IceOptions.BILLING_S3_BUCKET_NAME).split(",");
+        billingS3BucketRegions = properties.getProperty(IceOptions.BILLING_S3_BUCKET_REGION).split(",");
         billingS3BucketPrefixes = properties.getProperty(IceOptions.BILLING_S3_BUCKET_PREFIX, "").split(",");
         billingAccountIds = properties.getProperty(IceOptions.BILLING_PAYER_ACCOUNT_ID, "").split(",");
         billingAccessRoleNames = properties.getProperty(IceOptions.BILLING_ACCESS_ROLENAME, "").split(",");

src/java/com/netflix/ice/reader/ReaderConfig.java

@@ -73,7 +73,7 @@ public ReaderConfig(
         companyName = properties.getProperty(IceOptions.COMPANY_NAME, "");
         currencySign = properties.getProperty(IceOptions.CURRENCY_SIGN, "$");
         currencyRate = Double.parseDouble(properties.getProperty(IceOptions.CURRENCY_RATE, "1"));
-        highstockUrl = properties.getProperty(IceOptions.HIGHSTOCK_URL, "http://code.highcharts.com/stock/highstock.js");
+        highstockUrl = properties.getProperty(IceOptions.HIGHSTOCK_URL, "https://code.highcharts.com/stock/4.2.1/highstock.js");
 
         this.managers = managers;
         this.applicationGroupService = applicationGroupService;

src/java/com/netflix/ice/tag/Region.java

@@ -26,38 +26,53 @@
 public class Region extends Tag {
 
     public static final Region US_EAST_1 = new Region("us-east-1", "USE1");
+    public static final Region US_EAST_2 = new Region("us-east-2", "USE2");
     public static final Region US_WEST_1 = new Region("us-west-1", "USW1");
     public static final Region US_WEST_2 = new Region("us-west-2", "USW2");
     public static final Region EU_WEST_1 = new Region("eu-west-1", "EU");
+    public static final Region EU_WEST_2 = new Region("eu-west-2", "EUW2");
     public static final Region EU_CENTRAL_1 = new Region("eu-central-1", "EUC1");
+    public static final Region AP_SOUTH_1 = new Region("ap-south-1", "APS3");
     public static final Region AP_SOUTHEAST_1 = new Region("ap-southeast-1", "APS1");
     public static final Region AP_SOUTHEAST_2 = new Region("ap-southeast-2", "APS2");
     public static final Region AP_NORTHEAST_1 = new Region("ap-northeast-1","APN1");
+    public static final Region AP_NORTHEAST_2 = new Region("ap-northeast-2","APN2");
     public static final Region SA_EAST_1 = new Region("sa-east-1", "SAE1");
+    public static final Region CA_CENTRAL_1 = new Region("ca-central-1", "CAC1");
 
     private static ConcurrentMap<String, Region> regionsByName = Maps.newConcurrentMap();
     private static ConcurrentMap<String, Region> regionsByShortName = Maps.newConcurrentMap();
 
     static {
         regionsByShortName.put(US_EAST_1.shortName, US_EAST_1);
+        regionsByShortName.put(US_EAST_2.shortName, US_EAST_2);
         regionsByShortName.put(US_WEST_1.shortName, US_WEST_1);
         regionsByShortName.put(US_WEST_2.shortName, US_WEST_2);
         regionsByShortName.put(EU_WEST_1.shortName, EU_WEST_1);
+        regionsByShortName.put(EU_WEST_2.shortName, EU_WEST_2);
         regionsByShortName.put(EU_CENTRAL_1.shortName, EU_CENTRAL_1);
+        regionsByShortName.put(AP_SOUTH_1.shortName, AP_SOUTH_1);
         regionsByShortName.put(AP_SOUTHEAST_1.shortName, AP_SOUTHEAST_1);
         regionsByShortName.put(AP_SOUTHEAST_2.shortName, AP_SOUTHEAST_2);
         regionsByShortName.put(AP_NORTHEAST_1.shortName, AP_NORTHEAST_1);
+        regionsByShortName.put(AP_NORTHEAST_2.shortName, AP_NORTHEAST_2);
         regionsByShortName.put(SA_EAST_1.shortName, SA_EAST_1);
+        regionsByShortName.put(CA_CENTRAL_1.shortName, CA_CENTRAL_1);
 
         regionsByName.put(US_EAST_1.name, US_EAST_1);
+        regionsByName.put(US_EAST_2.name, US_EAST_2);
         regionsByName.put(US_WEST_1.name, US_WEST_1);
         regionsByName.put(US_WEST_2.name, US_WEST_2);
         regionsByName.put(EU_WEST_1.name, EU_WEST_1);
+        regionsByName.put(EU_WEST_2.name, EU_WEST_2);
         regionsByName.put(EU_CENTRAL_1.name, EU_CENTRAL_1);
+        regionsByName.put(AP_SOUTH_1.name, AP_SOUTH_1);
         regionsByName.put(AP_SOUTHEAST_1.name, AP_SOUTHEAST_1);
         regionsByName.put(AP_SOUTHEAST_2.name, AP_SOUTHEAST_2);
         regionsByName.put(AP_NORTHEAST_1.name, AP_NORTHEAST_1);
+        regionsByName.put(AP_NORTHEAST_2.name, AP_NORTHEAST_2);
         regionsByName.put(SA_EAST_1.name, SA_EAST_1);
+        regionsByName.put(CA_CENTRAL_1.name, CA_CENTRAL_1);
     }
 
     public final String shortName;