fuseParallelDos(col);
}
return;
}
}
ParallelDo p1 = (ParallelDo)output.getDeferredOp();
LazyCollection orig1 = (LazyCollection)p1.getOrigin();
if(orig1.isMaterialized()) {
return;
}
if(!(orig1.getDeferredOp() instanceof ParallelDo)) {
// Recursively apply this function to parent node
fuseParallelDos(orig1);
return;
}
// At this point we know ParallelDo fusion can be done -> Perform it
ParallelDo p2 = (ParallelDo)orig1.getDeferredOp();
// Lift combine values
if(p2 instanceof CombineValues) {
LazyCollection lCol = (LazyCollection)p2.getOrigin();
if(!lCol.isMaterialized() && lCol.getDeferredOp() instanceof GroupByKey) {
// Upper parallel do is CombineValues and follows a GroupByKey -> don't join
fuseParallelDos(orig1);
return;
}
}
final DoFn f1 = p1.getFunction();
final DoFn f2 = p2.getFunction();
// Define the joined function
DoFn newFn = new DoFn() {
@Override
public void process(Object v, final EmitFn emitter) {
f2.process(v, new EmitFn() {
@Override
public void emit(Object v) {
f1.process(v, emitter);
}
});
}
};
LazyCollection orig2 = (LazyCollection)p2.getOrigin();
ParallelDo newPDo = new ParallelDo(newFn, orig2, output);
// Clean & change pointers
orig2.downOps.remove(p2);
orig1.downOps.remove(p1);
orig2.addDownOp(newPDo);
output.deferredOp = newPDo;