diff --git a/.travis.yml b/.travis.yml index ac383008..d0577424 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,4 +11,5 @@ install: - yes | ./grailsw refresh-dependencies script: + - bash -n install.sh - ./grailsw compile \ No newline at end of file diff --git a/README.md b/README.md index 9b379e61..eeb16d03 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ Using basic setup, you don't need any extra code change and you will use the pro # Specify your currency conversion rate here. The default value is 1. If 1 pound = 1.5 dollar, then the rate is 0.6666667. ice.currencyRate=0.6666667 - 2.8 By default, Ice pulls in [Highstock](http://www.highcharts.com/) from its CDN. This CDN, unfortunately, does not (at present) support HTTPS. If you're serving Ice over HTTPS, your browser shouldn't or won't download Highstock from there. To fix this, you can serve Highstock somewhere else, and set this property: + 2.8 By default, Ice pulls in [Highstock](https://www.highcharts.com/) from its CDN. ice.highstockUrl=https://example.com/js/highstock.js @@ -263,6 +263,12 @@ Options with * require writing your own code. ice.use_blended=true +10. Extra Grails configuration file + + If you need to setup custom Grails settings, you can specify an additional configuration file to be loaded by Grails by setting the ``ice.config.location`` system property to the location of that file. + + See http://docs.grails.org/2.4.4/guide/single.html#configExternalized for more information. + ## Example IAM Permissions Grant the following permissions to either an instance role, or the user running the reports: diff --git a/grails-app/conf/Config.groovy b/grails-app/conf/Config.groovy index 518499b7..fe177004 100644 --- a/grails-app/conf/Config.groovy +++ b/grails-app/conf/Config.groovy @@ -17,13 +17,11 @@ // locations to search for config files that get merged into the main config // config files can either be Java properties files or ConfigSlurper scripts -grails.config.locations = [ +grails.config.locations = [] -] - -// if(System.properties["${appName}.config.location"]) { -// grails.config.locations << "file:" + System.properties["${appName}.config.location"] -// } +if(System.properties["${appName}.config.location"]) { + grails.config.locations << "file:" + System.properties["${appName}.config.location"] +} grails.project.groupId = appName // change this to alter the default package name and Maven publishing destination grails.mime.file.extensions = true // enables the parsing of file extensions from URLs into the request format diff --git a/install.sh b/install.sh index 0e18411f..223110f4 100755 --- a/install.sh +++ b/install.sh @@ -1,18 +1,27 @@ #!/bin/bash +# Exit the script if an error occur +set -e + # Get Ice in a ready-to-run state on a fresh AWI instance. -GRAILS_VERSION=2.2.1 +GRAILS_VERSION=2.4.4 # Install prerequisites - if [ -f /etc/redhat-release ]; then echo "Installing redhat packages" - sudo yum -y install git java-1.6.0-openjdk-devel.x86_64 wget unzip -else - [ -f /etc/debian-release ]; + sudo yum -y install git java-1.7.0-openjdk-devel.x86_64 wget unzip + +elif [[ -f /etc/debian-release || -f /etc/debian_version ]];then echo "Installing debian packages" - sudo apt-get -y install git openjdk-6-jdk wget unzip + sudo apt-get -y install git openjdk-7-jdk wget unzip + +elif [[ -f /etc/issue && $(grep "Amazon Linux AMI" /etc/issue) ]]; then + echo "Assuming AWS AMI, installing packages" + sudo yum -y install git java-1.7.0-openjdk-devel.x86_64 wget unzip + +else + echo "Unknown operating system. You may have to install Java 7 manually." fi INSTALL_DIR=$(pwd) @@ -53,11 +62,11 @@ fi grails ${JAVA_OPTS} wrapper # (Bug: Ice can't deal with this file existing and being empty.) -rm grails-app/i18n/messages.properties +rm -f grails-app/i18n/messages.properties # Create our local work directories (both for processing and reading) -mkdir ${HOME_DIR}/ice_processor -mkdir ${HOME_DIR}/ice_reader +mkdir -p ${HOME_DIR}/ice_processor +mkdir -p ${HOME_DIR}/ice_reader # Set up the config file cp src/java/sample.properties src/java/ice.properties @@ -74,9 +83,9 @@ do echo -n "-> " read -r PROCBUCKET done -sed -rie 's/=billing_s3bucketprefix\//=/; s|\/mnt\/|'"${HOME_DIR}"'\/|; s/=work_s3bucketprefix\//=/; s/^ice.account.*//; s/=billing_s3bucketname1/='${BILLBUCKET}'/; s/=work_s3bucketname/='${PROCBUCKET}'/' src/java/ice.properties +sed -ri 's/=billing_s3bucketprefix\//=/; s|\/mnt\/|'"${HOME_DIR}"'\/|; s/=work_s3bucketprefix\//=/; s/^ice.account.*//; s/=billing_s3bucketname1/='${BILLBUCKET}'/; s/=work_s3bucketname/='${PROCBUCKET}'/' src/java/ice.properties echo Ice is now ready to run as a processor. If you want to run the reader, edit: -echo ~/ice/src/java/ice.properties +echo "${INSTALL_DIR}/src/java/ice.properties" echo and alter the appropriate flags. You can now start Ice by running the following from the Ice root directory: echo ./grailsw -Djava.net.preferIPv4Stack=true -Dice.s3AccessKeyId=\ -Dice.s3SecretKey=\ run-app diff --git a/src/java/com/netflix/ice/basic/BasicDataManager.java b/src/java/com/netflix/ice/basic/BasicDataManager.java index b0ae1b0b..3882e163 100644 --- a/src/java/com/netflix/ice/basic/BasicDataManager.java +++ b/src/java/com/netflix/ice/basic/BasicDataManager.java @@ -65,7 +65,7 @@ public ReadOnlyData load(DateTime monthDate) throws Exception { public BasicDataManager(Product product, ConsolidateType consolidateType, boolean isCost) { this.product = product; this.consolidateType = consolidateType; - this.dbName = (isCost ? "cost_" : "usage_") + consolidateType + "_" + (product == null ? "all" : product.name); + this.dbName = (isCost ? "cost_" : "usage_") + consolidateType + "_" + (product == null ? "all" : product.s3Name); start(300); } diff --git a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java index dbb1ffb8..616dc90b 100644 --- a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java +++ b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java @@ -204,7 +204,7 @@ else if (result == Result.monthly) { } double resourceCostValue = costValue; - if (items.length > resourceIndex && !StringUtils.isEmpty(items[resourceIndex]) && config.resourceService != null) { + if (items.length > resourceIndex && config.resourceService != null) { if (config.useCostForResourceGroup.equals("modeled") && product == Product.ec2_instance) operation = Operation.getReservedInstances(config.reservationService.getDefaultReservationUtilization(0L)); diff --git a/src/java/com/netflix/ice/basic/BasicManagers.java b/src/java/com/netflix/ice/basic/BasicManagers.java index c74b6728..33968a64 100644 --- a/src/java/com/netflix/ice/basic/BasicManagers.java +++ b/src/java/com/netflix/ice/basic/BasicManagers.java @@ -25,6 +25,7 @@ import com.netflix.ice.processor.TagGroupWriter; import com.netflix.ice.reader.*; import com.netflix.ice.tag.Product; +import com.netflix.ice.tag.Tag; import java.util.Collection; import java.util.Map; @@ -99,6 +100,7 @@ private void doWork() { } else { String name = key.substring((config.workS3BucketPrefix + TagGroupWriter.DB_PREFIX).length()); + name = Tag.fromS3(name); product = config.productService.getProductByName(name); } if (!products.contains(product)) { diff --git a/src/java/com/netflix/ice/basic/BasicTagGroupManager.java b/src/java/com/netflix/ice/basic/BasicTagGroupManager.java index d3630e7e..5ef12379 100644 --- a/src/java/com/netflix/ice/basic/BasicTagGroupManager.java +++ b/src/java/com/netflix/ice/basic/BasicTagGroupManager.java @@ -48,7 +48,7 @@ public class BasicTagGroupManager extends Poller implements TagGroupManager { private Interval totalInterval; BasicTagGroupManager(Product product) { - this.dbName = TagGroupWriter.DB_PREFIX + (product == null ? "all" : product.name); + this.dbName = TagGroupWriter.DB_PREFIX + (product == null ? "all" : product.s3Name); file = new File(config.localDir, dbName); try { poll(); diff --git a/src/java/com/netflix/ice/common/AwsUtils.java b/src/java/com/netflix/ice/common/AwsUtils.java index 2efabd7c..f179a520 100755 --- a/src/java/com/netflix/ice/common/AwsUtils.java +++ b/src/java/com/netflix/ice/common/AwsUtils.java @@ -49,9 +49,9 @@ */ public class AwsUtils { private final static Logger logger = LoggerFactory.getLogger(AwsUtils.class); - private static Pattern billingFileWithTagsPattern = Pattern.compile(".+-aws-billing-detailed-line-items-with-resources-and-tags-(\\d\\d\\d\\d-\\d\\d).csv.zip"); - private static Pattern billingFileWithMonitoringPattern = Pattern.compile(".+-aws-billing-detailed-line-items-with-monitoring-(\\d\\d\\d\\d-\\d\\d).csv"); - private static Pattern billingFilePattern = Pattern.compile(".+-aws-billing-detailed-line-items-(\\d\\d\\d\\d-\\d\\d).csv.zip"); + private static Pattern billingFileWithTagsPattern = Pattern.compile(".+-aws-billing-detailed-line-items-with-resources-and-tags-(?:[A-Z]+-)?(\\d\\d\\d\\d-\\d\\d).csv.zip"); + private static Pattern billingFileWithMonitoringPattern = Pattern.compile(".+-aws-billing-detailed-line-items-with-monitoring-(?:[A-Z]+-)?(\\d\\d\\d\\d-\\d\\d).csv"); + private static Pattern billingFilePattern = Pattern.compile(".+-aws-billing-detailed-line-items-(?:[A-Z]+-)?(\\d\\d\\d\\d-\\d\\d).csv.zip"); public static final DateTimeFormatter monthDateFormat = DateTimeFormat.forPattern("yyyy-MM").withZone(DateTimeZone.UTC); public static final DateTimeFormatter dayDateFormat = DateTimeFormat.forPattern("yyyy-MM-dd").withZone(DateTimeZone.UTC); public static final DateTimeFormatter dateFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HHa").withZone(DateTimeZone.UTC); diff --git a/src/java/com/netflix/ice/common/IceOptions.java b/src/java/com/netflix/ice/common/IceOptions.java index 28c9ed99..bfb1ee87 100644 --- a/src/java/com/netflix/ice/common/IceOptions.java +++ b/src/java/com/netflix/ice/common/IceOptions.java @@ -40,8 +40,7 @@ public class IceOptions { public static final String CURRENCY_RATE = "ice.currencyRate"; /** - * The URL of highstock.js. The default value is the Highcharts CDN; change this if you need to - * serve it from somewhere else (for example, if you need HTTPS). + * The URL of highstock.js. The default value is the Highcharts CDN (HTTPS) */ public static final String HIGHSTOCK_URL = "ice.highstockUrl"; diff --git a/src/java/com/netflix/ice/processor/BillingFileProcessor.java b/src/java/com/netflix/ice/processor/BillingFileProcessor.java index ac907265..3d21790b 100644 --- a/src/java/com/netflix/ice/processor/BillingFileProcessor.java +++ b/src/java/com/netflix/ice/processor/BillingFileProcessor.java @@ -429,7 +429,7 @@ private void archive() throws Exception { private void archiveHourly(Map dataMap, String prefix) throws Exception { DateTime monthDateTime = new DateTime(startMilli, DateTimeZone.UTC); for (Product product: dataMap.keySet()) { - String prodName = product == null ? "all" : product.name; + String prodName = product == null ? "all" : product.s3Name; DataWriter writer = new DataWriter(prefix + "hourly_" + prodName + "_" + AwsUtils.monthDateFormat.print(monthDateTime), false); writer.archive(dataMap.get(product)); } @@ -448,7 +448,7 @@ private void archiveSummary(Map dataMap, String prefix) for (Product product: dataMap.keySet()) { - String prodName = product == null ? "all" : product.name; + String prodName = product == null ? "all" : product.s3Name; ReadWriteData data = dataMap.get(product); Collection tagGroups = data.getTagGroups(); @@ -758,4 +758,3 @@ private class BillingFile { } } } - diff --git a/src/java/com/netflix/ice/processor/DataWriter.java b/src/java/com/netflix/ice/processor/DataWriter.java index 875843d8..39b62bf4 100644 --- a/src/java/com/netflix/ice/processor/DataWriter.java +++ b/src/java/com/netflix/ice/processor/DataWriter.java @@ -19,6 +19,7 @@ import com.netflix.ice.common.AwsUtils; import com.netflix.ice.common.TagGroup; +import com.netflix.ice.tag.Tag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +39,7 @@ public class DataWriter { DataWriter(String name, boolean loadData) throws Exception { + name = Tag.toS3(name); dbName = name; file = new File(config.localDir, dbName); if (loadData) { @@ -81,4 +83,3 @@ void archive(ReadWriteData data) throws IOException { logger.info(this.dbName + " uploading done."); } } - diff --git a/src/java/com/netflix/ice/processor/TagGroupWriter.java b/src/java/com/netflix/ice/processor/TagGroupWriter.java index 35d8a6ef..41802c01 100644 --- a/src/java/com/netflix/ice/processor/TagGroupWriter.java +++ b/src/java/com/netflix/ice/processor/TagGroupWriter.java @@ -20,6 +20,7 @@ import com.google.common.collect.Maps; import com.netflix.ice.common.AwsUtils; import com.netflix.ice.common.TagGroup; +import com.netflix.ice.tag.Tag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +39,7 @@ public class TagGroupWriter { TagGroupWriter(String name) throws Exception { + name = Tag.toS3(name); dbName = DB_PREFIX + name; file = new File(config.localDir, dbName); logger.info("creating TagGroupWriter for " + file); @@ -74,4 +76,3 @@ void archive(Long monthMilli,Collection tagGroups) throws IOException logger.info(dbName + " uploading done."); } } - diff --git a/src/java/com/netflix/ice/tag/Tag.java b/src/java/com/netflix/ice/tag/Tag.java index b2fe9330..4973ceb6 100644 --- a/src/java/com/netflix/ice/tag/Tag.java +++ b/src/java/com/netflix/ice/tag/Tag.java @@ -28,8 +28,10 @@ public int compareTo(Tag t) { }; public final String name; + public final String s3Name; Tag(String name) { this.name = name; + this.s3Name = Tag.toS3(name); } @Override @@ -40,6 +42,22 @@ public boolean equals(Object o) { return false; } + /** + * Normalize a tagname suitable to be an S3 Filename + */ + public static String toS3(String name) { + name = name.replaceAll("/","--"); + return name; + } + + /** + * Normalize a tagname from an S3 Filename + */ + public static String fromS3(String name) { + name = name.replaceAll("--","/"); + return name; + } + @Override public String toString() { return this.name; diff --git a/src/java/com/netflix/ice/tag/Zone.java b/src/java/com/netflix/ice/tag/Zone.java index aa90ffa6..26ee6a8e 100644 --- a/src/java/com/netflix/ice/tag/Zone.java +++ b/src/java/com/netflix/ice/tag/Zone.java @@ -39,6 +39,7 @@ private Zone (Region region, String name) { public static final Zone US_EAST_1C = new Zone(Region.US_EAST_1, "us-east-1c"); public static final Zone US_EAST_1D = new Zone(Region.US_EAST_1, "us-east-1d"); public static final Zone US_EAST_1E = new Zone(Region.US_EAST_1, "us-east-1e"); + public static final Zone US_EAST_1F = new Zone(Region.US_EAST_1, "us-east-1f"); public static final Zone US_EAST_2A = new Zone(Region.US_EAST_2, "us-east-2a"); public static final Zone US_EAST_2B = new Zone(Region.US_EAST_2, "us-east-2b"); @@ -100,6 +101,7 @@ private Zone (Region region, String name) { zonesByName.put(US_EAST_1C.name, US_EAST_1C); zonesByName.put(US_EAST_1D.name, US_EAST_1D); zonesByName.put(US_EAST_1E.name, US_EAST_1E); + zonesByName.put(US_EAST_1F.name, US_EAST_1F); zonesByName.put(US_EAST_2A.name, US_EAST_2A); zonesByName.put(US_EAST_2B.name, US_EAST_2B); diff --git a/src/java/sample.properties b/src/java/sample.properties index bcb3053c..e5b7077a 100644 --- a/src/java/sample.properties +++ b/src/java/sample.properties @@ -13,7 +13,7 @@ ice.reservationPeriod=threeyear # default reservation utilization, possible values are LIGHT, MEDIUM, HEAVY. If you have both (LIGHT or MEDIUM) and HEAVY RIs, make sure you do not put HEAVY here. ice.reservationUtilization=MEDIUM -# the highstock url; host it somewhere else and change this if you need HTTPS +# the highstock url ice.highstockUrl=https://code.highcharts.com/stock/4.2.1/highstock.js # url prefix, e.g. http://ice.netflix.com/. Will be used in alert emails.