stores = PlanHelper.getStores(php);
int noJobs = stores.size();
int failedJobs = 0;
PigStats stats = new PigStats();
stats.setPhysicalPlan(php);
stats.setExecType(pc.getExecType());
for (POStore op : stores) {
op.setStoreImpl(new LocalPOStoreImpl(pc));
op.setUp();
}
// We need to handle stores that have loads as successors
// first. PlanHelper's getStores has returned those in the
// dependency order, so that's how we will run them.
for (Iterator<POStore> it = stores.iterator(); it.hasNext(); ) {
POStore op = it.next();
List<PhysicalOperator> sucs = new ArrayList<PhysicalOperator>();
if (php.getSuccessors(op) != null) {
sucs.addAll(php.getSuccessors(op));
}
if (sucs.size() != 0) {
log.info("running store with dependencies");
POStore[] st = new POStore[1];
st[0] = op;
failedJobs += runPipeline(st, pc);
for (PhysicalOperator suc: sucs) {
php.disconnect(op, suc);
}
it.remove();
}
}
// The remaining stores can be run together.
failedJobs += runPipeline(stores.toArray(new POStore[0]), pc);
stats.accumulateStats();
UDFFinishVisitor finisher = new UDFFinishVisitor(php, new DependencyOrderWalker<PhysicalOperator, PhysicalPlan>(php));
finisher.visit();
for (FileSpec spec: failedStores) {
log.info("Failed to produce result in: \""+spec.getFileName()+"\"");
}
for (FileSpec spec: succeededStores) {
log.info("Successfully stored result in: \""+spec.getFileName()+"\"");
}
UDFContext.getUDFContext().reset();
if (failedJobs == 0) {
log.info("Records written : " + stats.getRecordsWritten());
log.info("Bytes written : " + stats.getBytesWritten());
log.info("100% complete!");
log.info("Success!!");
return stats;
} else {
log.info("Failed jobs!!");