Examples of BulkIterationPlanNode


Examples of eu.stratosphere.compiler.plan.BulkIterationPlanNode

    }
   
    // 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
    if (terminationCriterion == null) {
      for (PlanNode candidate : candidates) {
        BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getPactContract().getName()+")", in, pspn, candidate);
        GlobalProperties gProps = candidate.getGlobalProperties().clone();
        LocalProperties lProps = candidate.getLocalProperties().clone();
        node.initProperties(gProps, lProps);
        target.add(node);
      }
    }
    else if(candidates.size() > 0) {
      List<PlanNode> terminationCriterionCandidates = this.terminationCriterion.getAlternativePlans(estimator);

      SingleRootJoiner singleRoot = (SingleRootJoiner) this.singleRoot;
     
      for (PlanNode candidate : candidates) {
        for(PlanNode terminationCandidate : terminationCriterionCandidates) {
          if (singleRoot.areBranchCompatible(candidate, terminationCandidate)) {
            BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getPactContract().getName()+")", in, pspn, candidate, terminationCandidate);
            GlobalProperties gProps = candidate.getGlobalProperties().clone();
            LocalProperties lProps = candidate.getLocalProperties().clone();
            node.initProperties(gProps, lProps);
            target.add(node);
           
          }
        }
      }
View Full Code Here

Examples of eu.stratosphere.compiler.plan.BulkIterationPlanNode

    else if (node instanceof SourcePlanNode) {
      TypeInformation<?> typeInfo = getTypeInfoFromSource((SourcePlanNode) node);
      ((SourcePlanNode) node).setSerializer(createSerializer(typeInfo));
    }
    else if (node instanceof BulkIterationPlanNode) {
      BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;

      if (iterationNode.getRootOfStepFunction() instanceof NAryUnionPlanNode) {
        throw new CompilerException("Optimizer cannot compile an iteration step function where next partial solution is created by a Union node.");
      }
     
      // traverse the termination criterion for the first time. create schema only, no utilities. Needed in case of intermediate termination criterion
      if (iterationNode.getRootOfTerminationCriterion() != null) {
        SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
        traverseChannel(addMapper.getInput());
      }

      BulkIterationBase<?> operator = (BulkIterationBase<?>) iterationNode.getPactContract();

      // set the serializer
      iterationNode.setSerializerForIterationChannel(createSerializer(operator.getOperatorInfo().getOutputType()));

      // done, we can now propagate our info down
      traverseChannel(iterationNode.getInput());
      traverse(iterationNode.getRootOfStepFunction());
    }
    else if (node instanceof WorksetIterationPlanNode) {
      WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
     
      if (iterationNode.getNextWorkSetPlanNode() instanceof NAryUnionPlanNode) {
        throw new CompilerException("Optimizer cannot compile a workset iteration step function where the next workset is produced by a Union node.");
      }
      if (iterationNode.getSolutionSetDeltaPlanNode() instanceof NAryUnionPlanNode) {
        throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node.");
      }
     
      DeltaIterationBase<?, ?> operator = (DeltaIterationBase<?, ?>) iterationNode.getPactContract();
     
      // set the serializers and comparators for the workset iteration
      iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType()));
      iterationNode.setWorksetSerializer(createSerializer(operator.getOperatorInfo().getSecondInputType()));
      iterationNode.setSolutionSetComparator(createComparator(operator.getOperatorInfo().getFirstInputType(),
          iterationNode.getSolutionSetKeyFields(), getSortOrders(iterationNode.getSolutionSetKeyFields(), null)));
     
      // traverse the inputs
      traverseChannel(iterationNode.getInput1());
      traverseChannel(iterationNode.getInput2());
     
      // traverse the step function
      traverse(iterationNode.getSolutionSetDeltaPlanNode());
      traverse(iterationNode.getNextWorkSetPlanNode());
    }
    else if (node instanceof SingleInputPlanNode) {
      SingleInputPlanNode sn = (SingleInputPlanNode) node;
     
      if (!(sn.getOptimizerNode().getPactContract() instanceof SingleInputOperator)) {
View Full Code Here

Examples of eu.stratosphere.compiler.plan.BulkIterationPlanNode

      }
      else if (node instanceof SourcePlanNode) {
        vertex = createDataSourceVertex((SourcePlanNode) node);
      }
      else if (node instanceof BulkIterationPlanNode) {
        BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;
        // for the bulk iteration, we skip creating anything for now. we create the graph
        // for the step function in the post visit.
       
        // check that the root of the step function has the same DOP as the iteration.
        // because the tail must have the same DOP as the head, we can only merge the last
        // operator with the tail, if they have the same DOP. not merging is currently not
        // implemented
        PlanNode root = iterationNode.getRootOfStepFunction();
        if (root.getDegreeOfParallelism() != node.getDegreeOfParallelism() ||
            root.getSubtasksPerInstance() != node.getSubtasksPerInstance())
        {
          throw new CompilerException("Error: The final operator of the step " +
              "function has a different degree of parallelism than the iteration operator itself.");
        }
       
        IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
        this.iterations.put(iterationNode, descr);
        vertex = null;
      }
      else if (node instanceof WorksetIterationPlanNode) {
        WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;

        // we have the same constraints as for the bulk iteration
        PlanNode nextWorkSet = iterationNode.getNextWorkSetPlanNode();
        PlanNode solutionSetDelta  = iterationNode.getSolutionSetDeltaPlanNode();
       
        if (nextWorkSet.getDegreeOfParallelism() != node.getDegreeOfParallelism() ||
          nextWorkSet.getSubtasksPerInstance() != node.getSubtasksPerInstance())
        {
          throw new CompilerException("It is currently not supported that the final operator of the step " +
              "function has a different degree of parallelism than the iteration operator itself.");
        }
        if (solutionSetDelta.getDegreeOfParallelism() != node.getDegreeOfParallelism() ||
          solutionSetDelta.getSubtasksPerInstance() != node.getSubtasksPerInstance())
        {
          throw new CompilerException("It is currently not supported that the final operator of the step " +
              "function has a different degree of parallelism than the iteration operator itself.");
        }
       
        IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
        this.iterations.put(iterationNode, descr);
        vertex = null;
      }
      else if (node instanceof SingleInputPlanNode) {
        vertex = createSingleInputVertex((SingleInputPlanNode) node);
      }
      else if (node instanceof DualInputPlanNode) {
        vertex = createDualInputVertex((DualInputPlanNode) node);
      }
      else if (node instanceof NAryUnionPlanNode) {
        // skip the union for now
        vertex = null;
      }
      else if (node instanceof BulkPartialSolutionPlanNode) {
        // create a head node (or not, if it is merged into its successor)
        vertex = createBulkIterationHead((BulkPartialSolutionPlanNode) node);
      }
      else if (node instanceof SolutionSetPlanNode) {
        // this represents an access into the solution set index.
        // we do not create a vertex for the solution set here (we create the head at the workset place holder)
       
        // we adjust the joins / cogroups that go into the solution set here
        for (Channel c : node.getOutgoingChannels()) {
          DualInputPlanNode target = (DualInputPlanNode) c.getTarget();
          AbstractJobVertex accessingVertex = this.vertices.get(target);
          TaskConfig conf = new TaskConfig(accessingVertex.getConfiguration());
          int inputNum = c == target.getInput1() ? 0 : c == target.getInput2() ? 1 : -1;
         
          // sanity checks
          if (inputNum == -1) {
            throw new CompilerException();
          }
         
          // adjust the driver
          if (conf.getDriver().equals(MatchDriver.class)) {
            conf.setDriver(inputNum == 0 ? JoinWithSolutionSetFirstDriver.class : JoinWithSolutionSetSecondDriver.class);
          }
          else if (conf.getDriver().equals(CoGroupDriver.class)) {
            conf.setDriver(inputNum == 0 ? CoGroupWithSolutionSetFirstDriver.class : CoGroupWithSolutionSetSecondDriver.class);
          }
          else {
            throw new CompilerException("Found join with solution set using incompatible operator (only Join/CoGroup are valid).");
          }
        }
       
        // make sure we do not visit this node again. for that, we add a 'already seen' entry into one of the sets
        this.chainedTasks.put(node, ALREADY_VISITED_PLACEHOLDER);
       
        vertex = null;
      }
      else if (node instanceof WorksetPlanNode) {
        // create the iteration head here
        vertex = createWorksetIterationHead((WorksetPlanNode) node);
      }
      else {
        throw new CompilerException("Unrecognized node type: " + node.getClass().getName());
      }
    }
    catch (Exception e) {
      throw new CompilerException("Error translating node '" + node + "': " + e.getMessage(), e);
    }
   
    // check if a vertex was created, or if it was chained or skipped
    if (vertex != null) {
      // set degree of parallelism
      int pd = node.getDegreeOfParallelism();
      vertex.setNumberOfSubtasks(pd);
 
      // check whether this is the vertex with the highest degree of parallelism
      if (this.maxDegreeVertex == null || this.maxDegreeVertex.getNumberOfSubtasks() < pd) {
        this.maxDegreeVertex = vertex;
      }
 
      // set the number of tasks per instance
      if (node.getSubtasksPerInstance() >= 1) {
        vertex.setNumberOfSubtasksPerInstance(node.getSubtasksPerInstance());
      }
     
      // check whether this vertex is part of an iteration step function
      if (this.currentIteration != null) {
        // check that the task has the same DOP as the iteration as such
        PlanNode iterationNode = (PlanNode) this.currentIteration;
        if (iterationNode.getDegreeOfParallelism() < pd) {
          throw new CompilerException("Error: All functions that are part of an iteration must have the same, or a lower, degree-of-parallelism than the iteration operator.");
        }
        if (iterationNode.getSubtasksPerInstance() < node.getSubtasksPerInstance()) {
          throw new CompilerException("Error: All functions that are part of an iteration must have the same, or a lower, number of subtasks-per-node than the iteration operator.");
        }
       
        // store the id of the iterations the step functions participate in
        IterationDescriptor descr = this.iterations.get(this.currentIteration);
View Full Code Here

Examples of eu.stratosphere.compiler.plan.BulkIterationPlanNode

    }
    else if (inputPlanNode instanceof BulkPartialSolutionPlanNode) {
      if (this.vertices.get(inputPlanNode) == null) {
        // merged iteration head
        final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) inputPlanNode;
        final BulkIterationPlanNode iterationNode = pspn.getContainingIterationNode();
       
        // check if the iteration's input is a union
        if (iterationNode.getInput().getSource() instanceof NAryUnionPlanNode) {
          allInChannels = ((NAryUnionPlanNode) iterationNode.getInput().getSource()).getInputs();
        } else {
          allInChannels = Collections.singletonList(iterationNode.getInput()).iterator();
        }
       
        // also, set the index of the gate with the partial solution
        targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex);
      } else {
        // standalone iteration head
        allInChannels = Collections.singletonList(input).iterator();
      }
    } else if (inputPlanNode instanceof WorksetPlanNode) {
      if (this.vertices.get(inputPlanNode) == null) {
        // merged iteration head
        final WorksetPlanNode wspn = (WorksetPlanNode) inputPlanNode;
        final WorksetIterationPlanNode iterationNode = wspn.getContainingIterationNode();
       
        // check if the iteration's input is a union
        if (iterationNode.getInput2().getSource() instanceof NAryUnionPlanNode) {
          allInChannels = ((NAryUnionPlanNode) iterationNode.getInput2().getSource()).getInputs();
        } else {
          allInChannels = Collections.singletonList(iterationNode.getInput2()).iterator();
        }
       
        // also, set the index of the gate with the partial solution
        targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex);
      } else {
View Full Code Here

Examples of eu.stratosphere.compiler.plan.BulkIterationPlanNode

        }
      }
      // cannot chain the nodes that produce the next workset in a bulk iteration if a termination criterion follows
      if (this.currentIteration != null && this.currentIteration instanceof BulkIterationPlanNode)
      {
        BulkIterationPlanNode wspn = (BulkIterationPlanNode) this.currentIteration;
        if (node == wspn.getRootOfTerminationCriterion() && wspn.getRootOfStepFunction() == pred){
          chaining = false;
        }else if(node.getOutgoingChannels().size() > 0 &&(wspn.getRootOfStepFunction() == pred ||
            wspn.getRootOfTerminationCriterion() == pred)) {
          chaining = false;
        }
      }
    }
   
View Full Code Here

Examples of eu.stratosphere.compiler.plan.BulkIterationPlanNode

    return vertex;
  }
 
  private JobTaskVertex createBulkIterationHead(BulkPartialSolutionPlanNode pspn) {
    // get the bulk iteration that corresponds to this partial solution node
    final BulkIterationPlanNode iteration = pspn.getContainingIterationNode();
   
    // check whether we need an individual vertex for the partial solution, or whether we
    // attach ourselves to the vertex of the parent node. We can combine the head with a node of
    // the step function, if
    // 1) There is one parent that the partial solution connects to via a forward pattern and no
    //    local strategy
    // 2) DOP and the number of subtasks per instance does not change
    // 3) That successor is not a union
    // 4) That successor is not itself the last node of the step function
    // 5) There is no local strategy on the edge for the initial partial solution, as
    //    this translates to a local strategy that would only be executed in the first iteration
   
    final boolean merge;
    if (mergeIterationAuxTasks && pspn.getOutgoingChannels().size() == 1) {
      final Channel c = pspn.getOutgoingChannels().get(0);
      final PlanNode successor = c.getTarget();
      merge = c.getShipStrategy() == ShipStrategyType.FORWARD &&
          c.getLocalStrategy() == LocalStrategy.NONE &&
          c.getTempMode() == TempMode.NONE &&
          successor.getDegreeOfParallelism() == pspn.getDegreeOfParallelism() &&
          successor.getSubtasksPerInstance() == pspn.getSubtasksPerInstance() &&
          !(successor instanceof NAryUnionPlanNode) &&
          successor != iteration.getRootOfStepFunction() &&
          iteration.getInput().getLocalStrategy() == LocalStrategy.NONE;
    } else {
      merge = false;
    }
   
    // create or adopt the head vertex
    final JobTaskVertex toReturn;
    final JobTaskVertex headVertex;
    final TaskConfig headConfig;
    if (merge) {
      final PlanNode successor = pspn.getOutgoingChannels().get(0).getTarget();
      headVertex = (JobTaskVertex) this.vertices.get(successor);
     
      if (headVertex == null) {
        throw new CompilerException(
          "Bug: Trying to merge solution set with its sucessor, but successor has not been created.");
      }
     
      // reset the vertex type to iteration head
      headVertex.setTaskClass(IterationHeadPactTask.class);
      headConfig = new TaskConfig(headVertex.getConfiguration());
      toReturn = null;
    } else {
      // instantiate the head vertex and give it a no-op driver as the driver strategy.
      // everything else happens in the post visit, after the input (the initial partial solution)
      // is connected.
      headVertex = new JobTaskVertex("PartialSolution ("+iteration.getNodeName()+")", this.jobGraph);
      headVertex.setTaskClass(IterationHeadPactTask.class);
      headConfig = new TaskConfig(headVertex.getConfiguration());
      headConfig.setDriver(NoOpDriver.class);
      toReturn = headVertex;
    }
View Full Code Here

Examples of eu.stratosphere.compiler.plan.BulkIterationPlanNode

    }
  }
 
  private void finalizeBulkIteration(IterationDescriptor descr) {
   
    final BulkIterationPlanNode bulkNode = (BulkIterationPlanNode) descr.getIterationNode();
    final JobTaskVertex headVertex = descr.getHeadTask();
    final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
    final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig();
   
    // ------------ finalize the head config with the final outputs and the sync gate ------------
    final int numStepFunctionOuts = headConfig.getNumOutputs();
    final int numFinalOuts = headFinalOutputConfig.getNumOutputs();
    headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig);
    headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts);
    final long memForBackChannel = bulkNode.getMemoryPerSubTask();
    if (memForBackChannel <= 0) {
      throw new CompilerException("Bug: No memory has been assigned to the iteration back channel.");
    }
    headConfig.setBackChannelMemory(memForBackChannel);
   
    // --------------------------- create the sync task ---------------------------
    final JobOutputVertex sync = new JobOutputVertex("Sync(" +
          bulkNode.getNodeName() + ")", this.jobGraph);
    sync.setOutputClass(IterationSynchronizationSinkTask.class);
    sync.setNumberOfSubtasks(1);
    this.auxVertices.add(sync);
   
    final TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getNumberOfSubtasks());

    // set the number of iteration / convergence criterion for the sync
    final int maxNumIterations = bulkNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
    if (maxNumIterations < 1) {
      throw new CompilerException("Cannot create bulk iteration with unspecified maximum number of iterations.");
    }
    syncConfig.setNumberOfIterations(maxNumIterations);
   
    // connect the sync task
    try {
      headVertex.connectTo(sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);
    } catch (JobGraphDefinitionException e) {
      throw new CompilerException("Bug: Cannot connect head vertex to sync task.");
    }
   
   
    // ----------------------------- create the iteration tail ------------------------------
   
    final PlanNode rootOfTerminationCriterion = bulkNode.getRootOfTerminationCriterion();
    final PlanNode rootOfStepFunction = bulkNode.getRootOfStepFunction();
    final TaskConfig tailConfig;
   
    JobTaskVertex rootOfStepFunctionVertex = (JobTaskVertex) this.vertices.get(rootOfStepFunction);
    if (rootOfStepFunctionVertex == null) {
      // last op is chained
      final TaskInChain taskInChain = this.chainedTasks.get(rootOfStepFunction);
      if (taskInChain == null) {
        throw new CompilerException("Bug: Tail of step function not found as vertex or chained task.");
      }
      rootOfStepFunctionVertex = (JobTaskVertex) taskInChain.getContainingVertex();

      // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
      tailConfig = taskInChain.getTaskConfig();
    } else {
      tailConfig = new TaskConfig(rootOfStepFunctionVertex.getConfiguration());
    }
   
    tailConfig.setIsWorksetUpdate();
   
    // No following termination criterion
    if(rootOfStepFunction.getOutgoingChannels().isEmpty()) {
     
      rootOfStepFunctionVertex.setTaskClass(IterationTailPactTask.class);
     
      tailConfig.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
      tailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
     
      // create the fake output task
      JobOutputVertex fakeTail = new JobOutputVertex("Fake Tail", this.jobGraph);
      fakeTail.setOutputClass(FakeOutputTask.class);
      fakeTail.setNumberOfSubtasks(headVertex.getNumberOfSubtasks());
      fakeTail.setNumberOfSubtasksPerInstance(headVertex.getNumberOfSubtasksPerInstance());
      this.auxVertices.add(fakeTail);
     
      // connect the fake tail
      try {
        rootOfStepFunctionVertex.connectTo(fakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
      } catch (JobGraphDefinitionException e) {
        throw new CompilerException("Bug: Cannot connect iteration tail vertex fake tail task");
      }
     
    }
   
   
    // create the fake output task for termination criterion, if needed
    final TaskConfig tailConfigOfTerminationCriterion;
    // If we have a termination criterion and it is not an intermediate node
    if(rootOfTerminationCriterion != null && rootOfTerminationCriterion.getOutgoingChannels().isEmpty()) {
      JobTaskVertex rootOfTerminationCriterionVertex = (JobTaskVertex) this.vertices.get(rootOfTerminationCriterion);
     
     
      if (rootOfTerminationCriterionVertex == null) {
        // last op is chained
        final TaskInChain taskInChain = this.chainedTasks.get(rootOfTerminationCriterion);
        if (taskInChain == null) {
          throw new CompilerException("Bug: Tail of termination criterion not found as vertex or chained task.");
        }
        rootOfTerminationCriterionVertex = (JobTaskVertex) taskInChain.getContainingVertex();

        // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
        tailConfigOfTerminationCriterion = taskInChain.getTaskConfig();
      } else {
        tailConfigOfTerminationCriterion = new TaskConfig(rootOfTerminationCriterionVertex.getConfiguration());
      }
     
      rootOfTerminationCriterionVertex.setTaskClass(IterationTailPactTask.class);
      // Hack
      tailConfigOfTerminationCriterion.setIsSolutionSetUpdate();
      tailConfigOfTerminationCriterion.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
      tailConfigOfTerminationCriterion.addOutputShipStrategy(ShipStrategyType.FORWARD);
     
      JobOutputVertex fakeTailTerminationCriterion = new JobOutputVertex("Fake Tail for Termination Criterion", this.jobGraph);
      fakeTailTerminationCriterion.setOutputClass(FakeOutputTask.class);
      fakeTailTerminationCriterion.setNumberOfSubtasks(headVertex.getNumberOfSubtasks());
      fakeTailTerminationCriterion.setNumberOfSubtasksPerInstance(headVertex.getNumberOfSubtasksPerInstance());
      this.auxVertices.add(fakeTailTerminationCriterion);
   
      // connect the fake tail
      try {
        rootOfTerminationCriterionVertex.connectTo(fakeTailTerminationCriterion, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
      } catch (JobGraphDefinitionException e) {
        throw new CompilerException("Bug: Cannot connect iteration tail vertex fake tail task for termination criterion");
      }
     
      // tell the head that it needs to wait for the solution set updates
      headConfig.setWaitForSolutionSetUpdate();
    }
   
    // ------------------- register the aggregators -------------------
    AggregatorRegistry aggs = bulkNode.getIterationNode().getIterationContract().getAggregators();
    Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
   
    headConfig.addIterationAggregators(allAggregators);
    syncConfig.addIterationAggregators(allAggregators);
   
View Full Code Here

Examples of eu.stratosphere.compiler.plan.BulkIterationPlanNode

        ((SourcePlanNode) node).setSerializer(createSerializer(parentSchema, node));
        // nothing else to be done here. the source has no input and no strategy itself
      }
    }
    else if (node instanceof BulkIterationPlanNode) {
      BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;
     
      // get the nodes current schema
      T schema;
      if (iterationNode.postPassHelper == null) {
        schema = createEmptySchema();
        iterationNode.postPassHelper = schema;
      } else {
        schema = (T) iterationNode.postPassHelper;
      }
      schema.increaseNumConnectionsThatContributed();
     
      // add the parent schema to the schema
      if (propagateParentSchemaDown) {
        addSchemaToSchema(parentSchema, schema, iterationNode.getPactContract().getName());
      }
     
      // check whether all outgoing channels have not yet contributed. come back later if not.
      if (schema.getNumConnectionsThatContributed() < iterationNode.getOutgoingChannels().size()) {
        return;
      }
     
      if (iterationNode.getRootOfStepFunction() instanceof NAryUnionPlanNode) {
        throw new CompilerException("Optimizer cannot compile an iteration step function where next partial solution is created by a Union node.");
      }
     
      // traverse the termination criterion for the first time. create schema only, no utilities. Needed in case of intermediate termination criterion
      if (iterationNode.getRootOfTerminationCriterion() != null) {
        SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
        traverse(addMapper.getInput().getSource(), createEmptySchema(), false);
        try {
          addMapper.getInput().setSerializer(createSerializer(createEmptySchema()));
        } catch (MissingFieldTypeInfoException e) {
          throw new RuntimeException(e);
        }
      }
     
      // traverse the step function for the first time. create schema only, no utilities
      traverse(iterationNode.getRootOfStepFunction(), schema, false);
     
      T pss = (T) iterationNode.getPartialSolutionPlanNode().postPassHelper;
      if (pss == null) {
        throw new CompilerException("Error in Optimizer Post Pass: Partial solution schema is null after first traversal of the step function.");
      }
     
      // traverse the step function for the second time, taking the schema of the partial solution
      traverse(iterationNode.getRootOfStepFunction(), pss, createUtilities);
     
      if (iterationNode.getRootOfTerminationCriterion() != null) {
        SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
        traverse(addMapper.getInput().getSource(), createEmptySchema(), createUtilities);
        try {
          addMapper.getInput().setSerializer(createSerializer(createEmptySchema()));
        } catch (MissingFieldTypeInfoException e) {
          throw new RuntimeException(e);
        }
      }
     
      // take the schema from the partial solution node and add its fields to the iteration result schema.
      // input and output schema need to be identical, so this is essentially a sanity check
      addSchemaToSchema(pss, schema, iterationNode.getPactContract().getName());
     
      // set the serializer
      if (createUtilities) {
        iterationNode.setSerializerForIterationChannel(createSerializer(pss, iterationNode.getPartialSolutionPlanNode()));
      }
     
      // done, we can now propagate our info down
      try {
        propagateToChannel(schema, iterationNode.getInput(), createUtilities);
      } catch (MissingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '"
          + iterationNode.getPactContract().getName() + "'. Missing type information for key field " +
          e.getFieldNumber());
      }
    }
    else if (node instanceof WorksetIterationPlanNode) {
      WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
     
      // get the nodes current schema
      T schema;
      if (iterationNode.postPassHelper == null) {
        schema = createEmptySchema();
        iterationNode.postPassHelper = schema;
      } else {
        schema = (T) iterationNode.postPassHelper;
      }
      schema.increaseNumConnectionsThatContributed();
     
      // add the parent schema to the schema (which refers to the solution set schema)
      if (propagateParentSchemaDown) {
        addSchemaToSchema(parentSchema, schema, iterationNode.getPactContract().getName());
      }
     
      // check whether all outgoing channels have not yet contributed. come back later if not.
      if (schema.getNumConnectionsThatContributed() < iterationNode.getOutgoingChannels().size()) {
        return;
      }
      if (iterationNode.getNextWorkSetPlanNode() instanceof NAryUnionPlanNode) {
        throw new CompilerException("Optimizer cannot compile a workset iteration step function where the next workset is produced by a Union node.");
      }
      if (iterationNode.getSolutionSetDeltaPlanNode() instanceof NAryUnionPlanNode) {
        throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node.");
      }
     
      // traverse the step function
      // pass an empty schema to the next workset and the parent schema to the solution set delta
      // these first traversals are schema only
      traverse(iterationNode.getNextWorkSetPlanNode(), createEmptySchema(), false);
      traverse(iterationNode.getSolutionSetDeltaPlanNode(), schema, false);
     
      T wss = (T) iterationNode.getWorksetPlanNode().postPassHelper;
      T sss = (T) iterationNode.getSolutionSetPlanNode().postPassHelper;
     
      if (wss == null) {
        throw new CompilerException("Error in Optimizer Post Pass: Workset schema is null after first traversal of the step function.");
      }
      if (sss == null) {
        throw new CompilerException("Error in Optimizer Post Pass: Solution set schema is null after first traversal of the step function.");
      }
     
      // make the second pass and instantiate the utilities
      traverse(iterationNode.getNextWorkSetPlanNode(), wss, createUtilities);
      traverse(iterationNode.getSolutionSetDeltaPlanNode(), sss, createUtilities);
     
      // add the types from the solution set schema to the iteration's own schema. since
      // the solution set input and the result must have the same schema, this acts as a sanity check.
      try {
        for (Map.Entry<Integer, X> entry : sss) {
          Integer pos = entry.getKey();
          schema.addType(pos, entry.getValue());
        }
      } catch (ConflictingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Conflicting type information for field " + e.getFieldNumber()
          + " in node '" + iterationNode.getPactContract().getName() + "'. Contradicting types between the " +
          "result of the iteration and the solution set schema: " + e.getPreviousType() +
          " and " + e.getNewType() + ". Most probable cause: Invalid constant field annotations.");
      }
     
      // set the serializers and comparators
      if (createUtilities) {
        WorksetIterationNode optNode = iterationNode.getIterationNode();
        iterationNode.setWorksetSerializer(createSerializer(wss, iterationNode.getWorksetPlanNode()));
        iterationNode.setSolutionSetSerializer(createSerializer(sss, iterationNode.getSolutionSetPlanNode()));
        try {
          iterationNode.setSolutionSetComparator(createComparator(optNode.getSolutionSetKeyFields(), null, sss));
        } catch (MissingFieldTypeInfoException ex) {
          throw new CompilerPostPassException("Could not set up the solution set for workset iteration '" +
              optNode.getPactContract().getName() + "'. Missing type information for key field " + ex.getFieldNumber() + '.');
        }
      }
     
      // done, we can now propagate our info down
      try {
        propagateToChannel(schema, iterationNode.getInitialSolutionSetInput(), createUtilities);
        propagateToChannel(wss, iterationNode.getInitialWorksetInput(), createUtilities);
      } catch (MissingFieldTypeInfoException ex) {
        throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '"
          + iterationNode.getPactContract().getName() + "'. Missing type information for key field " +
          ex.getFieldNumber());
      }
    }
    else if (node instanceof SingleInputPlanNode) {
      SingleInputPlanNode sn = (SingleInputPlanNode) node;
View Full Code Here

Examples of eu.stratosphere.compiler.plan.BulkIterationPlanNode

    if (visitable instanceof BulkIterationPlanNode) {
     
      DeadlockPreventer dp = new DeadlockPreventer();
      List<PlanNode> planSinks = new ArrayList<PlanNode>(1);
     
      BulkIterationPlanNode pspn = (BulkIterationPlanNode) visitable;
      planSinks.add(pspn.getRootOfStepFunction());
     
      dp.resolveDeadlocks(planSinks);
     
    }
    else if (visitable instanceof WorksetIterationPlanNode) {
     
      DeadlockPreventer dp = new DeadlockPreventer();
      List<PlanNode> planSinks = new ArrayList<PlanNode>(2);
     
      WorksetIterationPlanNode pspn = (WorksetIterationPlanNode) visitable;
      planSinks.add(pspn.getSolutionSetDeltaPlanNode());
      planSinks.add(pspn.getNextWorkSetPlanNode());
     
      dp.resolveDeadlocks(planSinks);
     
    }
  }
View Full Code Here

Examples of eu.stratosphere.compiler.plan.BulkIterationPlanNode

    final SinkPlanNode sink = or.getNode(SINK);
    final SingleInputPlanNode reducer = or.getNode(REDUCER_NAME);
    final SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getPredecessor();
    final SingleInputPlanNode mapper = or.getNode(MAPPER_NAME);
   
    final BulkIterationPlanNode iter = or.getNode(ITERATION_NAME);
   
    // -------------------- outside the loop -----------------------
   
    // check the sink
    assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
    assertEquals(LocalStrategy.NONE, sink.getInput().getLocalStrategy());
   
    // check the iteration
    assertEquals(ShipStrategyType.FORWARD, iter.getInput().getShipStrategy());
    assertEquals(LocalStrategy.NONE, iter.getInput().getLocalStrategy());
   
   
    // -------------------- inside the loop -----------------------
   
    // check the mapper
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.