} // FOR
}
final PartitionEstimator p_estimator = new PartitionEstimator(args.catalogContext);
final MarkovCostModel thread_costmodels[][] = new MarkovCostModel[num_threads][num_partitions];
final ProfileMeasurement profilers[] = new ProfileMeasurement[num_threads];
final LinkedBlockingDeque<Pair<Integer, TransactionTrace>> queues[] = (LinkedBlockingDeque<Pair<Integer, TransactionTrace>>[]) new LinkedBlockingDeque<?>[num_threads];
for (int i = 0; i < num_threads; i++) {
profilers[i] = new ProfileMeasurement("ESTIMATION");
queues[i] = new LinkedBlockingDeque<Pair<Integer, TransactionTrace>>();
} // FOR
LOG.info(String.format("Estimating the accuracy of the MarkovGraphs using %d transactions [threads=%d]", args.workload.getTransactionCount(), num_threads));
LOG.info("THRESHOLDS: " + args.thresholds);
// QUEUING THREAD
final AtomicBoolean queued_all = new AtomicBoolean(false);
runnables.add(new Runnable() {
@Override
public void run() {
List<TransactionTrace> all_txns = new ArrayList<TransactionTrace>(args.workload.getTransactions());
Collections.shuffle(all_txns);
int ctr = 0;
for (TransactionTrace txn_trace : all_txns) {
// Make sure it goes to the right base partition
int partition = HStoreConstants.NULL_PARTITION_ID;
try {
partition = p_estimator.getBasePartition(txn_trace);
} catch (Exception ex) {
throw new RuntimeException(ex);
}
assert(partition != HStoreConstants.NULL_PARTITION_ID) : "Failed to get base partition for " + txn_trace + "\n" + txn_trace.debug(args.catalog_db);
if (base_partition != HStoreConstants.NULL_PARTITION_ID && base_partition != partition)
continue;
int queue_idx = (global ? ctr : partition) % num_threads;
queues[queue_idx].add(Pair.of(partition, txn_trace));
if (++ctr % marker == 0)
LOG.info(String.format("Queued %d/%d transactions", ctr, num_transactions));
} // FOR
queued_all.set(true);
// Poke all our threads just in case they finished
for (Thread t : processing_threads)
t.interrupt();
}
});
// PROCESSING THREADS
for (int i = 0; i < num_threads; i++) {
final int thread_id = i;
runnables.add(new Runnable() {
@Override
public void run() {
Thread self = Thread.currentThread();
processing_threads.add(self);
Pair<Integer, TransactionTrace> pair = null;
final Set<MarkovOptimization> penalty_groups = new HashSet<MarkovOptimization>();
final Set<Penalty> penalties = new HashSet<Penalty>();
ObjectHistogram<MarkovOptimization> proc_h = null;
ProfileMeasurement profiler = profilers[thread_id];
assert (profiler != null);
MarkovCostModel costmodels[] = thread_costmodels[thread_id];
for (int p = 0; p < num_partitions; p++) {
MarkovGraphsContainer markovs = (global ? thread_markovs[thread_id].get(MarkovUtil.GLOBAL_MARKOV_CONTAINER_ID) : thread_markovs[thread_id].get(p));
MarkovEstimator t_estimator = new MarkovEstimator(args.catalogContext, p_estimator, markovs);
costmodels[p] = new MarkovCostModel(args.catalogContext, p_estimator, t_estimator, args.thresholds);
if (force_fullpath)
thread_costmodels[thread_id][p].forceFullPathComparison();
if (force_regenerate)
thread_costmodels[thread_id][p].forceRegenerateMarkovEstimates();
} // FOR
int thread_ctr = 0;
while (true) {
try {
if (queued_all.get()) {
pair = queues[thread_id].poll();
} else {
pair = queues[thread_id].take();
// Steal work
if (pair == null) {
for (int i = 0; i < num_threads; i++) {
if (i == thread_id)
continue;
pair = queues[i].take();
if (pair != null)
break;
} // FOR
}
}
} catch (InterruptedException ex) {
continue;
}
if (pair == null)
break;
int partition = pair.getFirst();
TransactionTrace txn_trace = pair.getSecond();
Procedure catalog_proc = txn_trace.getCatalogItem(args.catalog_db);
total_h.put(catalog_proc);
if (debug.val)
LOG.debug(String.format("Processing %s [%d / %d]", txn_trace, thread_ctr, thread_ctr + queues[thread_id].size()));
proc_h = proc_penalties_h.get(catalog_proc);
if (proc_h == null) {
synchronized (proc_penalties_h) {
proc_h = proc_penalties_h.get(catalog_proc);
if (proc_h == null) {
proc_h = new ObjectHistogram<MarkovOptimization>();
proc_penalties_h.put(catalog_proc, proc_h);
}
} // SYNCH
}
double cost = 0.0d;
Throwable error = null;
try {
profiler.start();
if (skip_processing == false) {
cost = costmodels[partition].estimateTransactionCost(args.catalogContext, txn_trace);
}
} catch (Throwable ex) {
error = ex;
} finally {
profiler.stop();
}
if (error != null) {
failures.getAndIncrement();
String msg = "Failed to estimate transaction cost for " + txn_trace;
if (stop_on_error)
throw new RuntimeException(msg, error);
LOG.warn(msg, error);
continue;
}
if (cost > 0) {
penalty_groups.clear();
penalties.clear();
for (Penalty p : costmodels[partition].getLastPenalties()) {
penalty_groups.add(p.getGroup());
penalties.add(p);
} // FOR
proc_h.put(penalty_groups);
optimizations_h.put(penalty_groups);
penalties_h.put(penalties);
missed_h.put(catalog_proc);
} else {
accurate_h.put(catalog_proc);
}
int global_ctr = total.incrementAndGet();
if (global_ctr % marker == 0)
LOG.info(String.format("Processed %d/%d transactions %s", global_ctr, num_transactions, (failures.get() > 0 ? String.format("[failures=%d]", failures.get()) : "")));
thread_ctr++;
} // WHILE
LOG.info(String.format("Processing Thread Finished %d / %d", thread_finished.incrementAndGet(), num_threads));
}
});
} // FOR
ThreadUtil.runGlobalPool(runnables);
Map<Object, String> debugLabels = CatalogUtil.getHistogramLabels(args.catalog_db.getProcedures());
ObjectHistogram<Procedure> fastpath_h = new ObjectHistogram<Procedure>();
fastpath_h.setDebugLabels(debugLabels);
ObjectHistogram<Procedure> fullpath_h = new ObjectHistogram<Procedure>();
fullpath_h.setDebugLabels(debugLabels);
ProfileMeasurement total_time = new ProfileMeasurement("ESTIMATION");
for (int i = 0; i < num_threads; i++) {
for (int p = 0; p < num_partitions; p++) {
MarkovCostModel mc = thread_costmodels[i][p];
fastpath_h.put(mc.fast_path_counter);
fullpath_h.put(mc.full_path_counter);
total_time.appendTime(profilers[i]);
}
} // FOR
int accurate_cnt = total.get() - (int) missed_h.getSampleCount();
assert (accurate_cnt == accurate_h.getSampleCount());
// ---------------------------------------------------------------------------------------------
Map<String, Object> m0 = new ListOrderedMap<String, Object>();
m0.put("PARTITIONS", num_partitions);
m0.put("FORCE FULLPATH", force_fullpath);
m0.put("FORCE REGENERATE", force_regenerate);
m0.put("COMPUTATION TIME", String.format("%.2f ms total / %.2f ms avg", total_time.getTotalThinkTimeMS(), total_time.getAverageThinkTimeMS()));
m0.put("TRANSACTION COUNTS", total_h.setDebugLabels(debugLabels));
Map<String, Object> m1 = new ListOrderedMap<String, Object>();
m1.put("ACCURATE TRANSACTIONS", accurate_h.setDebugLabels(debugLabels));
m1.put("MISSED TRANSACTIONS", missed_h.setDebugLabels(debugLabels));