package scalyr.utilities;
import java.io.File;
import java.util.List;
import java.util.Map;
import com.restartle.client.HttpClient;
import com.restartle.core.Dumper;
import com.restartle.core.IOUtil;
import com.restartle.core.Util;
import com.restartle.json.JsonArray;
import com.restartle.json.JsonObject;
import com.restartle.json.JsonUtil;
import com.restartle.log.L;
import com.scalyr.api.knobs.ConfigurationFile;
import com.scalyr.api.knobs.Knob;
/**
* Retrieves pricing data from cloud providers (Amazon, etc.) and generates a set of JSON tables in the files
* ec2.js, digitalocean.js, etc.
*
* NOTE: this class has dependencies on internal Scalyr utilities that are not currently released, so it is not
* possible to run as-is.
*/
public class CloudPriceFetcher {
/*
* Future extensions:
*
* - Alternate platforms (Windows, RHEL, etc.)
* - EBS optimized instances (http://aws.amazon.com/ec2/pricing/pricing-ebs-optimized-instances.json)
* - Record "I/O performance" for EC2 instances
* - Add an option to show, for each provider and server type, only the cheapest amortized option.
* Would filters out more-expensive reservation types and data centers.
*
* Notes:
*
* - EC2 Compute Units (ECUs) and Google GCEUs are treated as 1/2.75 cores.
* - Google "us-central1" and "us-central2" locations not listed separately, since prices are the same and
* specific locations not disclosed.
* - GCE "f1-micro" is listed as 0.5 GCEUs, arbitrarily.
* - Linode CPU interpreted as "1x priority" == 0.5 cores.
* - GCE instances can be rented in sub-hour increments.
*
* Sources:
*
* http://stackoverflow.com/questions/3636578/are-there-any-apis-for-amazon-web-services-pricing
* http://stackoverflow.com/questions/11253746/amazon-ec2-compute-unit-and-gceu-google-compute-engine-unit
* http://en.wikipedia.org/wiki/ENIAC
* http://en.wikipedia.org/wiki/Tianhe-2
*/
private static final double HOURS_PER_YEAR = Util.SECONDS_PER_YEAR / Util.SECONDS_PER_HOUR;
@SuppressWarnings("unused")
private static final double HOURS_PER_MONTH = HOURS_PER_YEAR / 12;
private static final String SOURCE_PATH = "/Users/steve/workspaces/forecloud/ScalyrSite/WebContent/cloud";
public static void main(String[] args) {
Knob.setDefaultFiles(new ConfigurationFile[0]);
L.initialize(true);
CloudPriceFetcher instance = new CloudPriceFetcher();
List<Tuple> optionsArray = Util.makeList();
instance.accumulateEC2Options(optionsArray);
emitOptions("ec2.js", optionsArray);
optionsArray = Util.makeList();
instance.accumulateDigitalOceanOptions(optionsArray);
emitOptions("digitalOcean.js", optionsArray);
optionsArray = Util.makeList();
instance.accumulateLinodeOptions(optionsArray);
emitOptions("linode.js", optionsArray);
optionsArray = Util.makeList();
instance.accumulateRackspaceOptions(optionsArray);
emitOptions("rackspace.js", optionsArray);
optionsArray = Util.makeList();
instance.accumulateGoogleOptions(optionsArray);
emitOptions("google.js", optionsArray);
optionsArray = Util.makeList();
instance.accumulateOtherOptions(optionsArray);
emitOptions("other.js", optionsArray);
// for (Tuple tuple : optionsArray)
// System.out.println("serverChoices.push(" + tuple.toJson() + ");");
// instance.dump("http://aws.amazon.com/ec2/pricing/json/mswin-ri-light.json", true, "light utilization reserved", "windows");
// instance.dump("http://aws.amazon.com/ec2/pricing/json/mswin-ri-medium.json", true, "medium utilization reserved", "windows");
// instance.dump("http://aws.amazon.com/ec2/pricing/json/mswin-ri-heavy.json", true, "heavy utilization reserved", "windows");
// Maybe-better version of data gathering at https://github.com/erans/ec2instancespricing/blob/master/ec2instancespricing.py
}
private static void emitOptions(String filename, List<Tuple> optionsArray) {
File file = new File(new File(SOURCE_PATH), filename);
StringBuilder sb = new StringBuilder();
sb.append("// Auto-generated by CloudPriceFetcher.java\n");
sb.append("\n");
for (Tuple tuple : optionsArray)
sb.append("serverChoices.push(" + tuple.toJson() + ");\n");
IOUtil.writeStringToFile(sb.toString(), file);
}
@SuppressWarnings("unused")
private void analyze(String url) {
JsonObject rawJson = fetchAndParse(url);
Dumper dumper = Dumper.stdout();
analyze(dumper, rawJson);
}
/**
* Describes a single, fully-specified purchase option.
*/
private static class Tuple {
/**
* Provider selling this option.
*/
public Provider provider;
/**
* Region in which this option is located.
*/
public Region region;
/**
* Provider's name for the specific data center or location. (Amazon calls this a "region", but we
* use that term for larger areas.)
*/
public String location;
/**
* Provider's name for the reservation type.
*/
public String reservationType;
/**
* Provider's name for the server / instance type.
*/
public String serverType;
/**
* Upfront payment for this option, in dollars.
*/
public double upfrontCost;
/**
* Hourly cost for this option, in dollars. Monthly prices are converted to hours assuming
* one month == 1/12 of a year.
*/
public double hourlyCost;
/**
* Life span of this option. Only relevant if upfrontCost is nonzero -- indicates what period of time we are
* purchasing for.
*/
public double termMonths;
/**
* Category into which this option's term falls.
*/
public Term term;
/**
* Number of CPU cores. (We treat Amazon ECUs as 1/2.75 cores.)
*/
public double cores;
/**
* RAM, in megabytes.
*/
public double ramMB;
/**
* Spinning disk, in megabytes.
*/
public double diskMB;
/**
* SSD storage, in megabytes.
*/
public double flashMB;
/**
* Included network bandwidth, in megabits per second. Monthly caps are converted to mbps assuming
* one month == 1/12 of a year.
*/
public double networkMbps;
public Tuple() {
}
public Tuple(Tuple src) {
provider = src.provider;
region = src.region;
location = src.location;
reservationType = src.reservationType;
serverType = src.serverType;
upfrontCost = src.upfrontCost;
hourlyCost = src.hourlyCost;
termMonths = src.termMonths;
term = src.term;
cores = src.cores;
ramMB = src.ramMB;
diskMB = src.diskMB;
flashMB = src.flashMB;
networkMbps = src.flashMB;
}
public JsonObject toJson() {
return new JsonObject()
.set("provider", provider.toString())
.set("region", region.toString())
.set("location", location)
.set("reservationType", reservationType)
.set("serverType", serverType)
.set("upfrontCost", upfrontCost)
.set("hourlyCost", hourlyCost)
.set("termMonths", termMonths)
.set("term", term.toString())
.set("cores", cores)
.set("ramMB", ramMB)
.set("diskMB", diskMB)
.set("flashMB", flashMB)
.set("networkMbps", networkMbps)
;
}
}
// Note that some region names (currently "N. America" are hard-coded in the JavaScript logic.
public static enum Region {
NorthAmerica { @Override public String toString() { return "N. America"; }},
SouthAmerica { @Override public String toString() { return "S. America"; }},
Europe,
Asia,
Africa,
Australia
};
// Note that some provider names (currently "Amazon", "Digital Ocean", and "Other") are hard-coded in the
// JavaScript logic.
public static enum Provider {
Amazon,
Rackspace,
Linode,
DigitalOcean { @Override public String toString() { return "Digital Ocean"; }},
Google,
Other
};
public static enum Term {
Hour { @Override public String toString() { return "Hour"; }},
Month { @Override public String toString() { return "Month"; }},
Year { @Override public String toString() { return "Year"; }},
ThreeYears { @Override public String toString() { return "3 Years"; }}
};
private void analyze(Dumper dumper, JsonObject json) {
dumper.line("keys: %", json.keySet());
dumper.indent();
for (Map.Entry<String, Object> entry : json.entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
if (value instanceof JsonObject) {
JsonObject obj = (JsonObject) value;
if (obj.prettyprint().length() <= 100) {
dumper.line("%: %", key, ((JsonObject) value).prettyprint());
} else {
dumper.line("%", key);
dumper.indent();
analyze(dumper, obj);
dumper.outdent();
}
} else if (value instanceof JsonArray) {
JsonArray ar = (JsonArray) value;
if (ar.size() == 0) {
dumper.line("%: []", key);
} else if (ar.get(0) instanceof JsonObject) {
dumper.line("%: array", key);
dumper.indent();
analyze(dumper, (JsonObject) ar.get(0));
dumper.outdent();
} else {
dumper.line("%: [%, ...]", key, ar.get(0));
}
} else {
dumper.line("%: %", key, value);
}
}
dumper.outdent();
}
/**
* Fetch JSON pricing data from Amazon. Add, optionsArray, a Tuple for each option listed in the JSON dump.
*
* @param optionsArray Array in which we accumulate the options.
*/
private void accumulateEC2Options(List<Tuple> optionsArray) {
Tuple baseTuple = new Tuple();
baseTuple.reservationType = "light";
accumulateEC2Options("http://aws.amazon.com/ec2/pricing/json/linux-ri-light.json", true, baseTuple, optionsArray);
baseTuple.reservationType = "medium";
accumulateEC2Options("http://aws.amazon.com/ec2/pricing/json/linux-ri-medium.json", true, baseTuple, optionsArray);
baseTuple.reservationType = "heavy";
accumulateEC2Options("http://aws.amazon.com/ec2/pricing/json/linux-ri-heavy.json", true, baseTuple, optionsArray);
baseTuple.reservationType = "hourly";
accumulateEC2Options("http://aws.amazon.com/ec2/pricing/pricing-on-demand-instances.json", false, baseTuple, optionsArray);
}
/**
* Fetch JSON pricing data from Amazon. Add, optionsArray, a Tuple for each option listed in the JSON dump.
*
* @param url URL of the JSON blob.
* @param isReservedInstance True if this JSON blob will describe reserved instances, false for on-demand.
* @param baseTuple Defines fields which are common to all options in this JSON blob. Currently, we do not
* expect any fields to be defined here. unless isReservedInstance is true, in which case reservationType
* should be populated but we will prefix "1 year " or "3 year ".
* @param optionsArray Array in which we accumulate the options.
*/
private void accumulateEC2Options(String url, boolean isReservedInstance, Tuple baseTuple, List<Tuple> optionsArray) {
JsonObject rawJson = fetchAndParse(url);
JsonObject config = rawJson.getJson("config");
for (JsonObject ec2Region : config.getArray("regions").getObjectIterable()) {
String ec2RegionName = ec2Region.getString("region");
for (JsonObject instanceType : ec2Region.getArray("instanceTypes").getObjectIterable()) {
String instanceTypeName = instanceType.getString("type");
for (JsonObject size : instanceType.getArray("sizes").getObjectIterable()) {
String sizeName = size.getString("size");
if (isReservedInstance) {
JsonObject map = columnsToMap(size.getArray("valueColumns"));
double upfront = getPrice(map, "yrTerm1"), hourly = getPrice(map, "yrTerm1Hourly");
if (upfront >= 0 && hourly >= 0) {
Tuple tuple = new Tuple(baseTuple);
parseAmazonRegion(ec2RegionName, tuple);
parseAmazonInstanceType(instanceTypeName, sizeName, tuple);
tuple.provider = Provider.Amazon;
tuple.upfrontCost = upfront;
tuple.hourlyCost = hourly;
tuple.termMonths = 12;
tuple.term = Term.Year;
tuple.networkMbps = 0;
tuple.reservationType = "1 year " + tuple.reservationType;
optionsArray.add(tuple);
}
upfront = getPrice(map, "yrTerm3");
hourly = getPrice(map, "yrTerm3Hourly");
if (upfront >= 0 && hourly >= 0) {
Tuple tuple = new Tuple(baseTuple);
parseAmazonRegion(ec2RegionName, tuple);
parseAmazonInstanceType(instanceTypeName, sizeName, tuple);
tuple.provider = Provider.Amazon;
tuple.upfrontCost = upfront;
tuple.hourlyCost = hourly;
tuple.termMonths = 36;
tuple.term = Term.ThreeYears;
tuple.networkMbps = 0;
tuple.reservationType = "3 year " + tuple.reservationType;
optionsArray.add(tuple);
}
} else {
for (JsonObject column : size.getArray("valueColumns").getObjectIterable()) {
String platformName = column.getString("name");
if ("N/A".equals(column.getJson("prices").get("USD")))
continue;
Tuple tuple = new Tuple(baseTuple);
parseAmazonRegion(ec2RegionName, tuple);
parseAmazonInstanceType(instanceTypeName, sizeName, tuple);
tuple.provider = Provider.Amazon;
tuple.reservationType = "on demand";
tuple.upfrontCost = 0;
tuple.hourlyCost = column.getJson("prices").getDouble("USD");
tuple.termMonths = 0;
tuple.term = Term.Hour;
tuple.networkMbps = 0;
optionsArray.add(tuple);
}
}
}
}
}
}
/**
* Set tuple.{region, location} for the specified region.
*
* @param ec2RegionName The region name, as found in Amazon's JSON pricing dumps.
*/
private void parseAmazonRegion(String ec2RegionName, Tuple tuple) {
if (ec2RegionName.equals("us-east")) {
tuple.region = Region.NorthAmerica;
tuple.location = "us-east-1 (Virginia)";
} else if (ec2RegionName.equals("us-west") || ec2RegionName.equals("us-west-1")) {
tuple.region = Region.NorthAmerica;
tuple.location = "us-west-1 (N. California)";
} else if (ec2RegionName.equals("us-west-2")) {
tuple.region = Region.NorthAmerica;
tuple.location = "us-west-2 (Oregon)";
} else if (ec2RegionName.equals("eu-west-1") || ec2RegionName.equals("eu-ireland")) {
tuple.region = Region.Europe;
tuple.location = "eu-west-1 (Ireland)";
} else if (ec2RegionName.equals("ap-southeast-1") || ec2RegionName.equals("apac-sin")) {
tuple.region = Region.Asia;
tuple.location = "ap-southeast-1 (Singapore)";
} else if (ec2RegionName.equals("ap-southeast-2") || ec2RegionName.equals("apac-syd")) {
tuple.region = Region.Australia;
tuple.location = "ap-southeast-2 (Sydney)";
} else if (ec2RegionName.equals("ap-northeast-1") || ec2RegionName.equals("apac-tokyo")) {
tuple.region = Region.Asia;
tuple.location = "ap-northeast-1 (Tokyo)";
} else if (ec2RegionName.equals("sa-east-1")) {
tuple.region = Region.SouthAmerica;
tuple.location = "sa-east-1 (Sao Paulo)";
} else {
throw new RuntimeException("Unknown Amazon region name: " + ec2RegionName);
}
}
/**
* Set tuple.{serverType, cores, ramMB, diskMB, flashMB} for the specified instance type.
*
* @param instanceType The instance type, as found in Amazon's JSON pricing dumps.
* @param instanceSize The instance size, as found in Amazon's JSON pricing dumps.
*/
private void parseAmazonInstanceType(String instanceType, String instanceSize, Tuple tuple) {
// Remove "ResI" or "ODI" (Reserved Instance or On-Remand Instance) from instanceType.
instanceType = instanceType.replace("ResI", "").replace("ODI", "");
String tag = instanceType + "/" + instanceSize;
if (tag.equals("generalPreviousGen/m1.small") || tag.equals("std/m1.small") || tag.equals("std/sm")) {
tuple.serverType = "m1.small";
tuple.cores = 1;
tuple.ramMB = 1.7 * 1024;
tuple.diskMB = 160 * 1024;
tuple.flashMB = 0;
} else if (tag.equals("generalPreviousGen/m1.medium") || tag.equals("std/m1.medium") || tag.equals("std/med")) {
tuple.serverType = "m1.medium";
tuple.cores = 2;
tuple.ramMB = 3.75 * 1024;
tuple.diskMB = 410 * 1024;
tuple.flashMB = 0;
} else if (tag.equals("generalPreviousGen/m1.large") || tag.equals("std/m1.large") || tag.equals("std/lg")) {
tuple.serverType = "m1.large";
tuple.cores = 4;
tuple.ramMB = 7.5 * 1024;
tuple.diskMB = 840 * 1024;
tuple.flashMB = 0;
} else if (tag.equals("generalPreviousGen/m1.xlarge") || tag.equals("std/m1.xlarge") || tag.equals("std/xl")) {
tuple.serverType = "m1.xlarge";
tuple.cores = 8;
tuple.ramMB = 15 * 1024;
tuple.diskMB = 1680 * 1024;
tuple.flashMB = 0;
} else if (tag.equals("generalCurrentGen/m3.xlarge") || tag.equals("secgenstd/m3.xlarge") || tag.equals("secgenstd/xl")) {
tuple.serverType = "m3.xlarge";
tuple.cores = 13;
tuple.ramMB = 15 * 1024;
tuple.diskMB = 0;
tuple.flashMB = 0;
} else if (tag.equals("generalCurrentGen/m3.2xlarge") || tag.equals("secgenstd/m3.2xlarge") || tag.equals("secgenstd/xxl")) {
tuple.serverType = "m3.2xlarge";
tuple.cores = 26;
tuple.ramMB = 30 * 1024;
tuple.diskMB = 0;
tuple.flashMB = 0;
} else if (tag.equals("uI/t1.micro") || tag.equals("u/t1.micro") || tag.equals("u/u")) {
tuple.serverType = "t1.micro";
tuple.cores = 0.5;
tuple.ramMB = 615;
tuple.diskMB = 0;
tuple.flashMB = 0;
} else if (tag.equals("hiMemCurrentGen/m2.xlarge") || tag.equals("hiMem/m2.xlarge") || tag.equals("hiMem/xl")) {
tuple.serverType = "m2.xlarge";
tuple.cores = 6.5;
tuple.ramMB = 17.1 * 1024;
tuple.diskMB = 420 * 1024;
tuple.flashMB = 0;
} else if (tag.equals("hiMemCurrentGen/m2.2xlarge") || tag.equals("hiMem/m2.2xlarge") || tag.equals("hiMem/xxl")) {
tuple.serverType = "m2.4xlarge";
tuple.cores = 13;
tuple.ramMB = 34.2 * 1024;
tuple.diskMB = 850 * 1024;
tuple.flashMB = 0;
} else if (tag.equals("hiMemCurrentGen/m2.4xlarge") || tag.equals("hiMem/m2.4xlarge") || tag.equals("hiMem/xxxxl")) {
tuple.serverType = "m2.2xlarge";
tuple.cores = 26;
tuple.ramMB = 68.3 * 1024;
tuple.diskMB = 1680 * 1024;
tuple.flashMB = 0;
} else if (tag.equals("computePreviousGen/c1.medium") || tag.equals("hiCPU/c1.medium") || tag.equals("hiCPU/med")) {
tuple.serverType = "c1.medium";
tuple.cores = 5;
tuple.ramMB = 1.7 * 1024;
tuple.diskMB = 350 * 1024;
tuple.flashMB = 0;
} else if (tag.equals("computePreviousGen/c1.xlarge") || tag.equals("hiCPU/c1.xlarge") || tag.equals("hiCPU/xl")) {
tuple.serverType = "c1.xlarge";
tuple.cores = 20;
tuple.ramMB = 7 * 1024;
tuple.diskMB = 1680 * 1024;
tuple.flashMB = 0;
} else if (tag.equals("c3/c3.large") || tag.equals("computeCurrentGen/c3.large")) {
tuple.serverType = "c3.large";
tuple.cores = 7;
tuple.ramMB = 3.75 * 1024;
tuple.diskMB = 0;
tuple.flashMB = 2 * 16 * 1024;
} else if (tag.equals("c3/c3.xlarge") || tag.equals("computeCurrentGen/c3.xlarge")) {
tuple.serverType = "c3.xlarge";
tuple.cores = 14;
tuple.ramMB = 7 * 1024;
tuple.diskMB = 0;
tuple.flashMB = 2 * 40 * 1024;
} else if (tag.equals("c3/c3.2xlarge") || tag.equals("computeCurrentGen/c3.2xlarge")) {
tuple.serverType = "c3.2xlarge";
tuple.cores = 28;
tuple.ramMB = 15 * 1024;
tuple.diskMB = 0;
tuple.flashMB = 2 * 80 * 1024;
} else if (tag.equals("c3/c3.4xlarge") || tag.equals("computeCurrentGen/c3.4xlarge")) {
tuple.serverType = "c3.4xlarge";
tuple.cores = 55;
tuple.ramMB = 30 * 1024;
tuple.diskMB = 0;
tuple.flashMB = 2 * 160 * 1024;
} else if (tag.equals("c3/c3.8xlarge") || tag.equals("computeCurrentGen/c3.8xlarge")) {
tuple.serverType = "c3.8xlarge";
tuple.cores = 108;
tuple.ramMB = 60 * 1024;
tuple.diskMB = 0;
tuple.flashMB = 2 * 320 * 1024;
} else if (tag.equals("clusterComputeI/cc1.4xlarge") || tag.equals("clusterComp/cc1.4xlarge")
|| tag.equals("clusterComputeI/xxxxl") || tag.equals("clusterComp/xxxxl")
|| tag.equals("clusterGPUI/xxxxl")) {
tuple.serverType = "cc1.4xlarge";
tuple.cores = 33.5;
tuple.ramMB = 22.5 * 1024;
tuple.diskMB = 3360 * 1024;
tuple.flashMB = 0;
} else if (tag.equals("computePreviousGen/cc2.8xlarge") || tag.equals("clusterComputeI/cc2.8xlarge") || tag.equals("clusterComp/cc2.8xlarge")
|| tag.equals("clusterComputeI/xxxxxxxxl") || tag.equals("clusterComp/xxxxxxxxl")) {
tuple.serverType = "cc2.8xlarge";
tuple.cores = 88;
tuple.ramMB = 60.5 * 1024;
tuple.diskMB = 4 * 840 * 1024;
tuple.flashMB = 0;
} else if (tag.equals("hiMemCurrentGen/cr1.8xlarge") || tag.equals("clusterHiMem/cr1.8xlarge") || tag.equals("clusterHiMem/xxxxxxxxl")) {
tuple.serverType = "cr1.8xlarge";
tuple.cores = 88;
tuple.ramMB = 244 * 1024;
tuple.diskMB = 0;
tuple.flashMB = 240 * 1024;
} else if (tag.equals("gpuCurrentGen/g2.2xlarge") || tag.equals("GPUI/g2.2xlarge")) {
tuple.serverType = "g2.2xlarge";
tuple.cores = 8;
tuple.ramMB = 15 * 1024;
tuple.diskMB = 0;
tuple.flashMB = 60 * 1000 / 1.024 / 1.024;
} else if (tag.equals("gpuPreviousGen/cg1.4xlarge") || tag.equals("hiIo/cg1.4xlarge")) {
tuple.serverType = "cg1.4xlarge";
tuple.cores = 33.5;
tuple.ramMB = 22.5 * 1024;
tuple.diskMB = 1680 * 1024;
tuple.flashMB = 0;
} else if (tag.equals("storagePreviousGen/hi1.4xlarge") || tag.equals("storageCurrentGen/hi1.4xlarge") || tag.equals("hiIo/hi1.4xlarge") || tag.equals("hiIo/xxxxl")) {
tuple.serverType = "hi1.4xlarge";
tuple.cores = 35;
tuple.ramMB = 60.5 * 1024;
tuple.diskMB = 0;
tuple.flashMB = 2048 * 1024;
} else if (tag.equals("storageCurrentGen/hs1.8xlarge") || tag.equals("hiStore/hs1.8xlarge") || tag.equals("hiStore/xxxxxxxxl")) {
tuple.serverType = "hs1.8xlarge";
tuple.cores = 35;
tuple.ramMB = 117 * 1024;
tuple.diskMB = 49152 * 1024;
tuple.flashMB = 0;
} else if (tag.equals("storageCurrentGen/i2.xlarge")) {
tuple.serverType = "i2.xlarge";
tuple.cores = 14;
tuple.ramMB = 30.5 * 1024;
tuple.diskMB = 0;
tuple.flashMB = 800 * 1024;
} else if (tag.equals("storageCurrentGen/i2.2xlarge")) {
tuple.serverType = "i2.2xlarge";
tuple.cores = 27;
tuple.ramMB = 61 * 1024;
tuple.diskMB = 0;
tuple.flashMB = 1600 * 1024;
} else if (tag.equals("storageCurrentGen/i2.4xlarge")) {
tuple.serverType = "i2.4xlarge";
tuple.cores = 53;
tuple.ramMB = 122 * 1024;
tuple.diskMB = 0;
tuple.flashMB = 3200 * 1024;
} else if (tag.equals("storageCurrentGen/i2.8xlarge")) {
tuple.serverType = "i2.8xlarge";
tuple.cores = 104;
tuple.ramMB = 244 * 1024;
tuple.diskMB = 0;
tuple.flashMB = 6400 * 1024;
} else {
throw new RuntimeException("Unknown Amazon instance type (" + instanceType + ") / size (" + instanceSize + ")");
}
tuple.cores /= 2.75; // Convert Amazon ECUs to "true" cores (approximately)
}
private JsonObject fetchAndParse(String url) {
String rawData = new HttpClient()
.setUrl(url)
.setTimeoutMs(20 * 1000)
.executeGet();
JsonObject rawJson = JsonUtil.parseJson(rawData);
return rawJson;
}
/**
* Given one of Amazon's "columns" arrays, return the same value as a map, indexed by the column's "name" attribute.
*/
private JsonObject columnsToMap(JsonArray columns) {
JsonObject result = new JsonObject();
for (JsonObject column : columns.getObjectIterable())
result.put(column.getString("name"), column);
return result;
}
private double getPrice(JsonObject map, String name) {
if ("N/A".equals(map.getJson(name).getJson("prices").get("USD")))
return -1;
return map.getJson(name).getJson("prices").getDouble("USD");
}
/**
* Add a Tuple to optionsArray for each Digital Ocean offering.
*/
private void accumulateDigitalOceanOptions(List<Tuple> optionsArray) {
// Source: https://www.digitalocean.com/pricing, Sept. 20, 2013
// One terabyte per month, converted to megabits per second.
double oneTerabyteInMbps = Math.pow(2.0, 40) * 8.0 / (Util.SECONDS_PER_YEAR / 12.0);
String[] locations = new String[]{"New York", "San Francisco", "Amsterdam"};
Region[] regions = new Region[]{Region.NorthAmerica, Region.NorthAmerica, Region.Europe};
for (int locationIndex = 0; locationIndex < locations.length; locationIndex++) {
Tuple baseTuple = new Tuple();
baseTuple.provider = Provider.DigitalOcean;
baseTuple.region = regions[locationIndex];
baseTuple.location = locations[locationIndex];
baseTuple.reservationType = "hourly";
baseTuple.diskMB = 0;
baseTuple.termMonths = 0;
baseTuple.term = Term.Hour;
baseTuple.upfrontCost = 0;
Tuple tuple = new Tuple(baseTuple);
tuple.serverType = "512MB";
tuple.hourlyCost = 5 / 672.0;
tuple.cores = 1;
tuple.ramMB = 512;
tuple.flashMB = 20 * 1024;
tuple.networkMbps = 1.0 * oneTerabyteInMbps;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "1GB";
tuple.hourlyCost = 10 / 672.0;
tuple.cores = 1;
tuple.ramMB = 1024;
tuple.flashMB = 30 * 1024;
tuple.networkMbps = 2.0 * oneTerabyteInMbps;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "2GB";
tuple.hourlyCost = 20 / 672.0;
tuple.cores = 2;
tuple.ramMB = 2 * 1024;
tuple.flashMB = 40 * 1024;
tuple.networkMbps = 3.0 * oneTerabyteInMbps;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "4GB";
tuple.hourlyCost = 40 / 672.0;
tuple.cores = 2;
tuple.ramMB = 4 * 1024;
tuple.flashMB = 60 * 1024;
tuple.networkMbps = 4.0 * oneTerabyteInMbps;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "8GB";
tuple.hourlyCost = 80 / 672.0;
tuple.cores = 4;
tuple.ramMB = 8 * 1024;
tuple.flashMB = 80 * 1024;
tuple.networkMbps = 5.0 * oneTerabyteInMbps;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "16GB";
tuple.hourlyCost = 160 / 672.0;
tuple.cores = 8;
tuple.ramMB = 16 * 1024;
tuple.flashMB = 160 * 1024;
tuple.networkMbps = 6.0 * oneTerabyteInMbps;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "32GB";
tuple.hourlyCost = 320 / 672.0;
tuple.cores = 12;
tuple.ramMB = 32 * 1024;
tuple.flashMB = 320 * 1024;
tuple.networkMbps = 7.0 * oneTerabyteInMbps;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "48GB";
tuple.hourlyCost = 480 / 672.0;
tuple.cores = 16;
tuple.ramMB = 48 * 1024;
tuple.flashMB = 480 * 1024;
tuple.networkMbps = 8.0 * oneTerabyteInMbps;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "64GB";
tuple.hourlyCost = 640 / 672.0;
tuple.cores = 20;
tuple.ramMB = 64 * 1024;
tuple.flashMB = 640 * 1024;
tuple.networkMbps = 9.0 * oneTerabyteInMbps;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "96GB";
tuple.hourlyCost = 960 / 672.0;
tuple.cores = 24;
tuple.ramMB = 96 * 1024;
tuple.flashMB = 960 * 1024;
tuple.networkMbps = 10.0 * oneTerabyteInMbps;
optionsArray.add(tuple);
}
}
/**
* Add a Tuple to optionsArray for various miscellaneous offerings.
*/
private void accumulateOtherOptions(List<Tuple> optionsArray) {
Tuple tuple = new Tuple();
tuple.provider = Provider.Other;
tuple.region = Region.NorthAmerica;
tuple.location = "Maryland";
tuple.reservationType = "purchase";
tuple.diskMB = 0;
tuple.termMonths = 98; // ENIAC was operational for a bit over eight years
tuple.term = Term.ThreeYears;
tuple.upfrontCost = 5900000;
tuple.serverType = "ENIAC";
tuple.hourlyCost = 15; // 150kW at $0.10 / kWh
tuple.cores = 0.001;
tuple.ramMB = 0;
tuple.flashMB = 0;
tuple.networkMbps = 0;
optionsArray.add(tuple);
tuple = new Tuple();
tuple.provider = Provider.Other;
tuple.region = Region.Asia;
tuple.location = "Guangzhou, China";
tuple.reservationType = "purchase";
tuple.diskMB = 12.4 * 1024 * 1024 * 1024;
tuple.termMonths = 36;
tuple.term = Term.ThreeYears;
tuple.upfrontCost = 390000000;
tuple.serverType = "Tianhe-2";
tuple.hourlyCost = 2400; // 24MW at $0.10 / kWh
tuple.cores = 3120000;
tuple.ramMB = 1375.0 * 1024 * 1024;
tuple.flashMB = 0;
tuple.networkMbps = 0;
optionsArray.add(tuple);
}
/**
* Add a Tuple to optionsArray for each Linode offering.
*/
private void accumulateLinodeOptions(List<Tuple> optionsArray) {
// Source: https://www.linode.com, Sept. 20, 2013
// One terabyte per month, converted to megabits per second.
double oneTerabyteInMbps = Math.pow(2.0, 40) * 8.0 / (Util.SECONDS_PER_YEAR / 12.0);
String[] locations = new String[]{"Tokyo", "London", "Newark", "Atlanta", "Dallas", "Fremont"};
Region[] regions = new Region[]{ Region.Asia, Region.Europe, Region.NorthAmerica,
Region.NorthAmerica, Region.NorthAmerica, Region.NorthAmerica };
for (int locationIndex = 0; locationIndex < locations.length; locationIndex++) {
Tuple baseTuple = new Tuple();
baseTuple.provider = Provider.Linode;
baseTuple.region = regions[locationIndex];
baseTuple.location = locations[locationIndex];
baseTuple.reservationType = "monthly";
baseTuple.flashMB = 0;
baseTuple.termMonths = 1;
baseTuple.term = Term.Month;
baseTuple.hourlyCost = 0;
Tuple tuple = new Tuple(baseTuple);
tuple.serverType = "1GB";
tuple.upfrontCost = 20;
tuple.cores = 0.5;
tuple.ramMB = 1024;
tuple.diskMB = 48 * 1024;
tuple.networkMbps = 2.0 * oneTerabyteInMbps;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "2GB";
tuple.upfrontCost = 40;
tuple.cores = 1.0;
tuple.ramMB = 2 * 1024;
tuple.diskMB = 96 * 1024;
tuple.networkMbps = 4.0 * oneTerabyteInMbps;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "4GB";
tuple.upfrontCost = 80;
tuple.cores = 2.0;
tuple.ramMB = 4 * 1024;
tuple.diskMB = 192 * 1024;
tuple.networkMbps = 8.0 * oneTerabyteInMbps;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "8GB";
tuple.upfrontCost = 160;
tuple.cores = 4.0;
tuple.ramMB = 8 * 1024;
tuple.diskMB = 384 * 1024;
tuple.networkMbps = 16.0 * oneTerabyteInMbps;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "16GB";
tuple.upfrontCost = 320;
tuple.cores = 8.0;
tuple.ramMB = 16 * 1024;
tuple.diskMB = 768 * 1024;
tuple.networkMbps = 20.0 * oneTerabyteInMbps;
optionsArray.add(tuple);
}
}
/**
* Add a Tuple to optionsArray for each Rackspace offering.
*/
private void accumulateRackspaceOptions(List<Tuple> optionsArray) {
// Source: http://www.rackspace.com/cloud/servers/pricing/, Nov. 5, 2013
String[] locations = new String[]{"Dallas", "Chicago", "Virginia", "London", "Hong Kong", "Sydney"};
Region[] regions = new Region[]{ Region.NorthAmerica, Region.NorthAmerica, Region.NorthAmerica,
Region.Europe, Region.Asia, Region.Australia };
for (int locationIndex = 0; locationIndex < locations.length; locationIndex++) {
Tuple baseTuple = new Tuple();
baseTuple.provider = Provider.Rackspace;
baseTuple.region = regions[locationIndex];
baseTuple.location = locations[locationIndex];
baseTuple.reservationType = "hourly";
baseTuple.diskMB = 0;
baseTuple.termMonths = 0;
baseTuple.term = Term.Hour;
baseTuple.upfrontCost = 0;
Tuple tuple = new Tuple(baseTuple);
tuple.serverType = "1GB Perf. 1";
tuple.hourlyCost = 0.04;
tuple.cores = 0.5;
tuple.ramMB = 1024;
tuple.flashMB = 20 * 1024;
tuple.networkMbps = 200;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "2GB Perf. 1";
tuple.hourlyCost = 0.08;
tuple.cores = 1;
tuple.ramMB = 2048;
tuple.flashMB = 60 * 1024;
tuple.networkMbps = 400;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "4GB Perf. 1";
tuple.hourlyCost = 0.16;
tuple.cores = 2;
tuple.ramMB = 4096;
tuple.flashMB = 80 * 1024;
tuple.networkMbps = 800;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "8GB Perf. 1";
tuple.hourlyCost = 0.32;
tuple.cores = 4;
tuple.ramMB = 8192;
tuple.flashMB = 120 * 1024;
tuple.networkMbps = 1600;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "15GB Perf. 2";
tuple.hourlyCost = 0.66;
tuple.cores = 2;
tuple.ramMB = 15 * 1024;
tuple.flashMB = 190 * 1024;
tuple.networkMbps = 1250;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "30GB Perf. 2";
tuple.hourlyCost = 1.36;
tuple.cores = 4;
tuple.ramMB = 30 * 1024;
tuple.flashMB = 340 * 1024;
tuple.networkMbps = 2500;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "60GB Perf. 2";
tuple.hourlyCost = 2.72;
tuple.cores = 8;
tuple.ramMB = 60 * 1024;
tuple.flashMB = 640 * 1024;
tuple.networkMbps = 5000;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "90GB Perf. 2";
tuple.hourlyCost = 4.08;
tuple.cores = 12;
tuple.ramMB = 90 * 1024;
tuple.flashMB = 940 * 1024;
tuple.networkMbps = 7500;
optionsArray.add(tuple);
tuple = new Tuple(baseTuple);
tuple.serverType = "120GB Perf. 2";
tuple.hourlyCost = 5.44;
tuple.cores = 16;
tuple.ramMB = 120 * 1024;
tuple.flashMB = 1240 * 1024;
tuple.networkMbps = 10000;
optionsArray.add(tuple);
}
}
/**
* Add a Tuple to optionsArray for each Google Compute Engine offering.
*/
private void accumulateGoogleOptions(List<Tuple> optionsArray) {
// Source: https://developers.google.com/compute/pricing#machinetype, Jan. 15, 2014
Tuple baseTuple = new Tuple();
baseTuple.provider = Provider.Google;
baseTuple.reservationType = "hourly";
baseTuple.flashMB = 0;
baseTuple.termMonths = 0;
baseTuple.term = Term.Hour;
baseTuple.upfrontCost = 0;
baseTuple.networkMbps = 0;
baseTuple.region = Region.NorthAmerica;
baseTuple.location = "US Central";
optionsArray.add(createGoogleTuple(baseTuple, "n1-standard-1", 2.75, 3.75, 0, 0.104));
optionsArray.add(createGoogleTuple(baseTuple, "n1-standard-2", 5.50, 7.50, 0, 0.207));
optionsArray.add(createGoogleTuple(baseTuple, "n1-standard-4", 11.0, 15.0, 0, 0.415));
optionsArray.add(createGoogleTuple(baseTuple, "n1-standard-8", 22.0, 30.0, 0, 0.829));
optionsArray.add(createGoogleTuple(baseTuple, "n1-standard-16", 44.0, 60.0, 0, 1.659));
optionsArray.add(createGoogleTuple(baseTuple, "f1-micro", 0.5, 0.60, 0, 0.019));
optionsArray.add(createGoogleTuple(baseTuple, "g1-small", 1.38, 1.70, 0, 0.054));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highmem-2", 5.50, 13.0, 0, 0.244));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highmem-4", 11.0, 26.0, 0, 0.488));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highmem-8", 22.0, 52.0, 0, 0.975));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highmem-16", 44.0, 104.0, 0, 1.951));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highcpu-2", 5.50, 1.80, 0, 0.131));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highcpu-4", 11.0, 3.60, 0, 0.261));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highcpu-8", 22.0, 7.20, 0, 0.522));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highcpu-16", 44.0, 14.40, 0, 1.044));
baseTuple.region = Region.Europe;
baseTuple.location = "Europe West";
optionsArray.add(createGoogleTuple(baseTuple, "n1-standard-1", 2.75, 3.75, 0, 0.114));
optionsArray.add(createGoogleTuple(baseTuple, "n1-standard-2", 5.50, 7.50, 0, 0.228));
optionsArray.add(createGoogleTuple(baseTuple, "n1-standard-4", 11.0, 15.0, 0, 0.456));
optionsArray.add(createGoogleTuple(baseTuple, "n1-standard-8", 22.0, 30.0, 0, 0.912));
optionsArray.add(createGoogleTuple(baseTuple, "n1-standard-16", 44.0, 60.0, 0, 1.825));
optionsArray.add(createGoogleTuple(baseTuple, "f1-micro", 0.5, 0.60, 0, 0.021));
optionsArray.add(createGoogleTuple(baseTuple, "g1-small", 1.38, 1.70, 0, 0.059));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highmem-2", 5.50, 13.0, 0, 0.275));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highmem-4", 11.0, 26.0, 0, 0.549));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highmem-8", 22.0, 52.0, 0, 1.098));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highmem-16", 44.0, 104.0, 0, 2.196));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highcpu-2", 5.50, 1.80, 0, 0.146));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highcpu-4", 11.0, 3.60, 0, 0.292));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highcpu-8", 22.0, 7.20, 0, 0.584));
optionsArray.add(createGoogleTuple(baseTuple, "n1-highcpu-16", 44.0, 14.40, 0, 1.167));
}
private Tuple createGoogleTuple(Tuple baseTuple, String configuration, double gceus, double ramGB, double diskGB, double dollarsPerHour) {
Tuple tuple = new Tuple(baseTuple);
tuple.serverType = configuration;
tuple.hourlyCost = dollarsPerHour;
tuple.cores = gceus / 2.75;
tuple.ramMB = ramGB * 1024;
tuple.diskMB = diskGB * 1024;
return tuple;
}
}