// check that the root of the step function has the same DOP as the iteration.
// because the tail must have the same DOP as the head, we can only merge the last
// operator with the tail, if they have the same DOP. not merging is currently not
// implemented
PlanNode root = iterationNode.getRootOfStepFunction();
if (root.getDegreeOfParallelism() != node.getDegreeOfParallelism() ||
root.getSubtasksPerInstance() != node.getSubtasksPerInstance())
{
throw new CompilerException("Error: The final operator of the step " +
"function has a different degree of parallelism than the iteration operator itself.");
}
IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
this.iterations.put(iterationNode, descr);
vertex = null;
}
else if (node instanceof WorksetIterationPlanNode) {
WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
// we have the same constraints as for the bulk iteration
PlanNode nextWorkSet = iterationNode.getNextWorkSetPlanNode();
PlanNode solutionSetDelta = iterationNode.getSolutionSetDeltaPlanNode();
if (nextWorkSet.getDegreeOfParallelism() != node.getDegreeOfParallelism() ||
nextWorkSet.getSubtasksPerInstance() != node.getSubtasksPerInstance())
{
throw new CompilerException("It is currently not supported that the final operator of the step " +
"function has a different degree of parallelism than the iteration operator itself.");
}
if (solutionSetDelta.getDegreeOfParallelism() != node.getDegreeOfParallelism() ||
solutionSetDelta.getSubtasksPerInstance() != node.getSubtasksPerInstance())
{
throw new CompilerException("It is currently not supported that the final operator of the step " +
"function has a different degree of parallelism than the iteration operator itself.");
}
IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
this.iterations.put(iterationNode, descr);
vertex = null;
}
else if (node instanceof SingleInputPlanNode) {
vertex = createSingleInputVertex((SingleInputPlanNode) node);
}
else if (node instanceof DualInputPlanNode) {
vertex = createDualInputVertex((DualInputPlanNode) node);
}
else if (node instanceof NAryUnionPlanNode) {
// skip the union for now
vertex = null;
}
else if (node instanceof BulkPartialSolutionPlanNode) {
// create a head node (or not, if it is merged into its successor)
vertex = createBulkIterationHead((BulkPartialSolutionPlanNode) node);
}
else if (node instanceof SolutionSetPlanNode) {
// this represents an access into the solution set index.
// we do not create a vertex for the solution set here (we create the head at the workset place holder)
// we adjust the joins / cogroups that go into the solution set here
for (Channel c : node.getOutgoingChannels()) {
DualInputPlanNode target = (DualInputPlanNode) c.getTarget();
AbstractJobVertex accessingVertex = this.vertices.get(target);
TaskConfig conf = new TaskConfig(accessingVertex.getConfiguration());
int inputNum = c == target.getInput1() ? 0 : c == target.getInput2() ? 1 : -1;
// sanity checks
if (inputNum == -1) {
throw new CompilerException();
}
// adjust the driver
if (conf.getDriver().equals(MatchDriver.class)) {
conf.setDriver(inputNum == 0 ? JoinWithSolutionSetFirstDriver.class : JoinWithSolutionSetSecondDriver.class);
}
else if (conf.getDriver().equals(CoGroupDriver.class)) {
conf.setDriver(inputNum == 0 ? CoGroupWithSolutionSetFirstDriver.class : CoGroupWithSolutionSetSecondDriver.class);
}
else {
throw new CompilerException("Found join with solution set using incompatible operator (only Join/CoGroup are valid).");
}
}
// make sure we do not visit this node again. for that, we add a 'already seen' entry into one of the sets
this.chainedTasks.put(node, ALREADY_VISITED_PLACEHOLDER);
vertex = null;
}
else if (node instanceof WorksetPlanNode) {
// create the iteration head here
vertex = createWorksetIterationHead((WorksetPlanNode) node);
}
else {
throw new CompilerException("Unrecognized node type: " + node.getClass().getName());
}
}
catch (Exception e) {
throw new CompilerException("Error translating node '" + node + "': " + e.getMessage(), e);
}
// check if a vertex was created, or if it was chained or skipped
if (vertex != null) {
// set degree of parallelism
int pd = node.getDegreeOfParallelism();
vertex.setNumberOfSubtasks(pd);
// check whether this is the vertex with the highest degree of parallelism
if (this.maxDegreeVertex == null || this.maxDegreeVertex.getNumberOfSubtasks() < pd) {
this.maxDegreeVertex = vertex;
}
// set the number of tasks per instance
if (node.getSubtasksPerInstance() >= 1) {
vertex.setNumberOfSubtasksPerInstance(node.getSubtasksPerInstance());
}
// check whether this vertex is part of an iteration step function
if (this.currentIteration != null) {
// check that the task has the same DOP as the iteration as such
PlanNode iterationNode = (PlanNode) this.currentIteration;
if (iterationNode.getDegreeOfParallelism() < pd) {
throw new CompilerException("Error: All functions that are part of an iteration must have the same, or a lower, degree-of-parallelism than the iteration operator.");
}
if (iterationNode.getSubtasksPerInstance() < node.getSubtasksPerInstance()) {
throw new CompilerException("Error: All functions that are part of an iteration must have the same, or a lower, number of subtasks-per-node than the iteration operator.");
}
// store the id of the iterations the step functions participate in
IterationDescriptor descr = this.iterations.get(this.currentIteration);