public PartitionPlan generate(DesignerHints hints) throws Exception {
final PartitionPlan pplan = new PartitionPlan();
// Generate an AccessGraph and select the column with the greatest
// weight for each table
final AccessGraph agraph = this.generateAccessGraph();
final boolean calculate_memory = (hints.force_replication_size_limit != null && hints.max_memory_per_partition != 0);
double total_partitionRatio = 0.0;
long total_partitionSize = 0l;
for (DesignerVertex v : agraph.getVertices()) {
Table catalog_tbl = v.getCatalogItem();
String table_key = CatalogKey.createKey(catalog_tbl);
Collection<Column> forced_columns = hints.getForcedTablePartitionCandidates(catalog_tbl);
TableStatistics ts = info.stats.getTableStatistics(catalog_tbl);
assert (ts != null) : "Null TableStatistics for " + catalog_tbl;
double partition_size = (calculate_memory ? (ts.tuple_size_total / (double) info.getNumPartitions()) : 0);
double partition_ratio = (calculate_memory ? (ts.tuple_size_total / (double) hints.max_memory_per_partition) : 0);
TableEntry pentry = null;
if (debug.val) {
Map<String, Object> m = new ListOrderedMap<String, Object>();
m.put("Read Only", ts.readonly);
m.put("Table Size", StringUtil.formatSize(ts.tuple_size_total));
m.put("Table Partition Size", StringUtil.formatSize((long)partition_size));
m.put("Table Partition Ratio", String.format("%.02f", partition_ratio));
m.put("Total Partition Size", String.format("%s / %s", StringUtil.formatSize(total_partitionSize), StringUtil.formatSize(hints.max_memory_per_partition)));
m.put("Total Partition Ratio", String.format("%.02f", total_partitionRatio));
LOG.debug(String.format("%s\n%s", catalog_tbl.getName(), StringUtil.formatMaps(m)));
}
// -------------------------------
// Replication
// -------------------------------
if (hints.force_replication.contains(table_key) || (calculate_memory && ts.readonly && hints.enable_replication_readonly && partition_ratio <= hints.force_replication_size_limit)) {
total_partitionRatio += partition_ratio;
total_partitionSize += ts.tuple_size_total;
Column catalog_col = ReplicatedColumn.get(catalog_tbl);
pentry = new TableEntry(PartitionMethodType.REPLICATION, catalog_col);
if (debug.val)
LOG.debug(String.format("Replicating %s at all partitions [%s]", catalog_tbl.getName(), catalog_col.fullName()));
// -------------------------------
// Forced Selection
// -------------------------------
} else if (forced_columns.isEmpty() == false) {
// Assume there is only one candidate
assert (forced_columns.size() == 1) : "Unexpected number of forced columns: " + forced_columns;
Column catalog_col = CollectionUtil.first(forced_columns);
pentry = new TableEntry(PartitionMethodType.HASH, catalog_col);
total_partitionRatio += partition_size / (double) hints.max_memory_per_partition;
total_partitionSize += partition_size;
if (debug.val)
LOG.debug(String.format("Forcing %s to be partitioned by specific column [%s]", catalog_tbl.getName(), catalog_col.fullName()));
// -------------------------------
// Select Most Popular
// -------------------------------
} else {
// If there are no edges, then we'll just randomly pick a column
// since it doesn't matter
final Collection<DesignerEdge> edges = agraph.getIncidentEdges(v);
if (edges.isEmpty())
continue;
if (trace.val)
LOG.trace(catalog_tbl + " has " + edges.size() + " edges in AccessGraph");
ObjectHistogram<Column> column_histogram = null;
ObjectHistogram<Column> join_column_histogram = new ObjectHistogram<Column>();
ObjectHistogram<Column> self_column_histogram = new ObjectHistogram<Column>();
// Map<Column, Double> unsorted = new HashMap<Column, Double>();
for (DesignerEdge e : edges) {
Collection<DesignerVertex> vertices = agraph.getIncidentVertices(e);
DesignerVertex v0 = CollectionUtil.get(vertices, 0);
DesignerVertex v1 = CollectionUtil.get(vertices, 1);
boolean self = (v0.equals(v) && v1.equals(v));
column_histogram = (self ? self_column_histogram : join_column_histogram);
double edge_weight = e.getTotalWeight();
PredicatePairs cset = e.getAttribute(AccessGraph.EdgeAttributes.COLUMNSET);
if (trace.val)
LOG.trace("Examining ColumnSet for " + e.toString(true));
Histogram<Column> cset_histogram = cset.buildHistogramForType(Column.class);
Collection<Column> columns = cset_histogram.values();
if (trace.val)
LOG.trace("Constructed Histogram for " + catalog_tbl + " from ColumnSet:\n"
+ cset_histogram.setDebugLabels(CatalogUtil.getHistogramLabels(cset_histogram.values())).toString(100, 50));
for (Column catalog_col : columns) {
if (!catalog_col.getParent().equals(catalog_tbl))
continue;
if (catalog_col.getNullable())
continue;
long cnt = cset_histogram.get(catalog_col);
if (trace.val)
LOG.trace("Found Match: " + catalog_col.fullName() + " [cnt=" + cnt + "]");
column_histogram.put(catalog_col, Math.round(cnt * edge_weight));
} // FOR
// System.err.println(cset.debug());
// LOG.info("[" + e.getTotalWeight() + "]: " + cset);
} // FOR
// If there were no join columns, then use the self-reference
// histogram
column_histogram = (join_column_histogram.isEmpty() ? self_column_histogram : join_column_histogram);
if (column_histogram.isEmpty()) {
EventObserver<DesignerVertex> observer = new EventObserver<DesignerVertex>() {
@Override
public void update(EventObservable<DesignerVertex> o, DesignerVertex v) {
for (DesignerEdge e : agraph.getIncidentEdges(v)) {
LOG.info(e.getAttribute(AccessGraph.EdgeAttributes.COLUMNSET));
}
LOG.info(StringUtil.repeat("-", 100));
}
};