Package eu.stratosphere.compiler.plan

Examples of eu.stratosphere.compiler.plan.Channel


          continue;
        }
       
        for (RequestedGlobalProperties igps1: intGlobal1) {
          // create a candidate channel for the first input. mark it cached, if the connection says so
          final Channel c1 = new Channel(child1, this.input1.getMaterializationMode());
          if (this.input1.getShipStrategy() == null) {
            // free to choose the ship strategy
            igps1.parameterizeChannel(c1, globalDopChange1, localDopChange1);
           
            // if the DOP changed, make sure that we cancel out properties, unless the
            // ship strategy preserves/establishes them even under changing DOPs
            if (globalDopChange1 && !c1.getShipStrategy().isNetworkStrategy()) {
              c1.getGlobalProperties().reset();
            }
            if (localDopChange1 && !(c1.getShipStrategy().isNetworkStrategy() ||
                  c1.getShipStrategy().compensatesForLocalDOPChanges())) {
              c1.getGlobalProperties().reset();
            }
          } else {
            // ship strategy fixed by compiler hint
            if (this.keys1 != null) {
              c1.setShipStrategy(this.input1.getShipStrategy(), this.keys1.toFieldList());
            } else {
              c1.setShipStrategy(this.input1.getShipStrategy());
            }
           
            if (globalDopChange1) {
              c1.adjustGlobalPropertiesForFullParallelismChange();
            } else if (localDopChange1) {
              c1.adjustGlobalPropertiesForLocalParallelismChange();
            }
          }
         
          for (RequestedGlobalProperties igps2: intGlobal2) {
            // create a candidate channel for the first input. mark it cached, if the connection says so
            final Channel c2 = new Channel(child2, this.input2.getMaterializationMode());
            if (this.input2.getShipStrategy() == null) {
              // free to choose the ship strategy
              igps2.parameterizeChannel(c2, globalDopChange2, localDopChange2);
             
              // if the DOP changed, make sure that we cancel out properties, unless the
              // ship strategy preserves/establishes them even under changing DOPs
              if (globalDopChange2 && !c2.getShipStrategy().isNetworkStrategy()) {
                c2.getGlobalProperties().reset();
              }
              if (localDopChange2 && !(c2.getShipStrategy().isNetworkStrategy() ||
                    c2.getShipStrategy().compensatesForLocalDOPChanges())) {
                c2.getGlobalProperties().reset();
              }
            } else {
              // ship strategy fixed by compiler hint
              if (this.keys2 != null) {
                c2.setShipStrategy(this.input2.getShipStrategy(), this.keys2.toFieldList());
              } else {
                c2.setShipStrategy(this.input2.getShipStrategy());
              }
             
              if (globalDopChange2) {
                c2.adjustGlobalPropertiesForFullParallelismChange();
              } else if (localDopChange2) {
                c2.adjustGlobalPropertiesForLocalParallelismChange();
              }
            }
           
            /* ********************************************************************
             * NOTE: Depending on how we proceed with different partitionings,
             *       we might at some point need a compatibility check between
             *       the pairs of global properties.
             * *******************************************************************/
           
            for (GlobalPropertiesPair gpp : allGlobalPairs) {
              if (gpp.getProperties1().isMetBy(c1.getGlobalProperties()) &&
                gpp.getProperties2().isMetBy(c2.getGlobalProperties()) )
              {
                // we form a valid combination, so create the local candidates
                // for this
                addLocalCandidates(c1, c2, broadcastPlanChannels, igps1, igps2, outputPlans, allLocalPairs, estimator);
                break;
View Full Code Here


  {
    final LocalProperties lp1 = template1.getLocalPropertiesAfterShippingOnly();
    final LocalProperties lp2 = template2.getLocalPropertiesAfterShippingOnly();
   
    for (RequestedLocalProperties ilp1 : this.input1.getInterestingProperties().getLocalProperties()) {
      final Channel in1 = template1.clone();
      if (ilp1.isMetBy(lp1)) {
        in1.setLocalStrategy(LocalStrategy.NONE);
      } else {
        ilp1.parameterizeChannel(in1);
      }
     
      for (RequestedLocalProperties ilp2 : this.input2.getInterestingProperties().getLocalProperties()) {
        final Channel in2 = template2.clone();
        if (ilp2.isMetBy(lp2)) {
          in2.setLocalStrategy(LocalStrategy.NONE);
        } else {
          ilp2.parameterizeChannel(in2);
        }
       
        allPossibleLoop:
        for (OperatorDescriptorDual dps: this.possibleProperties) {
          for (LocalPropertiesPair lpp : dps.getPossibleLocalProperties()) {
            if (lpp.getProperties1().isMetBy(in1.getLocalProperties()) &&
              lpp.getProperties2().isMetBy(in2.getLocalProperties()) )
            {
              // valid combination
              // for non trivial local properties, we need to check that they are co compatible
              // (such as when some sort order is requested, that both are the same sort order
              if (dps.areCoFulfilled(lpp.getProperties1(), lpp.getProperties2(),
                in1.getLocalProperties(), in2.getLocalProperties()))
              {
                // all right, co compatible
                instantiate(dps, in1, in2, broadcastPlanChannels, target, estimator, rgps1, rgps2, ilp1, ilp2);
                break allPossibleLoop;
              } else {
View Full Code Here

   
    InterestingProperties ips = this.input.getInterestingProperties();
    for (PlanNode p : subPlans) {
      for (RequestedGlobalProperties gp : ips.getGlobalProperties()) {
        for (RequestedLocalProperties lp : ips.getLocalProperties()) {
          Channel c = new Channel(p);
          gp.parameterizeChannel(c, globalDopChange, localDopChange);

          if (lp.isMetBy(c.getLocalPropertiesAfterShippingOnly())) {
            c.setLocalStrategy(LocalStrategy.NONE);
          } else {
            lp.parameterizeChannel(c);
          }
         
          // no need to check whether the created properties meet what we need in case
View Full Code Here

          // Chained Task. Sanity check first...
          final Iterator<Channel> inConns = node.getInputs();
          if (!inConns.hasNext()) {
            throw new CompilerException("Bug: Found chained task with no input.");
          }
          final Channel inConn = inConns.next();
         
          if (inConns.hasNext()) {
            throw new CompilerException("Bug: Found a chained task with more than one input!");
          }
          if (inConn.getLocalStrategy() != null && inConn.getLocalStrategy() != LocalStrategy.NONE) {
            throw new CompilerException("Bug: Found a chained task with an input local strategy.");
          }
          if (inConn.getShipStrategy() != null && inConn.getShipStrategy() != ShipStrategyType.FORWARD) {
            throw new CompilerException("Bug: Found a chained task with an input ship strategy other than FORWARD.");
          }
 
          AbstractJobVertex container = chainedTask.getContainingVertex();
         
          if (container == null) {
            final PlanNode sourceNode = inConn.getSource();
            container = this.vertices.get(sourceNode);
            if (container == null) {
              // predecessor is itself chained
              container = this.chainedTasks.get(sourceNode).getContainingVertex();
              if (container == null) {
                throw new IllegalStateException("Bug: Chained task predecessor has not been assigned its containing vertex.");
              }
            } else {
              // predecessor is a proper task job vertex and this is the first chained task. add a forward connection entry.
              new TaskConfig(container.getConfiguration()).addOutputShipStrategy(ShipStrategyType.FORWARD);
            }
            chainedTask.setContainingVertex(container);
          }
         
          // add info about the input serializer type
          chainedTask.getTaskConfig().setInputSerializer(inConn.getSerializer(), 0);
         
          // update name of container task
          String containerTaskName = container.getName();
          if(containerTaskName.startsWith("CHAIN ")) {
            container.setName(containerTaskName+" -> "+chainedTask.getTaskName());
          } else {
            container.setName("CHAIN "+containerTaskName+" -> "+chainedTask.getTaskName());
          }
         
          this.chainedTasksInSequence.add(chainedTask);
          return;
        }
        else if (node instanceof BulkPartialSolutionPlanNode ||
            node instanceof WorksetPlanNode)
        {
          // merged iteration head task. the task that the head is merged with will take care of it
          return;
        } else {
          throw new CompilerException("Bug: Unrecognized merged task vertex.");
        }
      }
     
      // -------- Here, we translate non-chained tasks -------------
     
      // create the config that will contain all the description of the inputs
      final TaskConfig targetVertexConfig = new TaskConfig(targetVertex.getConfiguration());
           
      // get the inputs. if this node is the head of an iteration, we obtain the inputs from the
      // enclosing iteration node, because the inputs are the initial inputs to the iteration.
      final Iterator<Channel> inConns;
      if (node instanceof BulkPartialSolutionPlanNode) {
        inConns = ((BulkPartialSolutionPlanNode) node).getContainingIterationNode().getInputs();
        // because the partial solution has its own vertex, is has only one (logical) input.
        // note this in the task configuration
        targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
      } else if (node instanceof WorksetPlanNode) {
        WorksetPlanNode wspn = (WorksetPlanNode) node;
        // input that is the initial workset
        inConns = Collections.singleton(wspn.getContainingIterationNode().getInput2()).iterator();
       
        // because we have a stand-alone (non-merged) workset iteration head, the initial workset will
        // be input 0 and the solution set will be input 1
        targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
        targetVertexConfig.setIterationHeadSolutionSetInputIndex(1);
      } else {
        inConns = node.getInputs();
      }
      if (!inConns.hasNext()) {
        throw new CompilerException("Bug: Found a non-source task with no input.");
      }
     
      int inputIndex = 0;
      while (inConns.hasNext()) {
        Channel input = inConns.next();
        inputIndex += translateChannel(input, inputIndex, targetVertex, targetVertexConfig, false);
      }
      // broadcast variables
      int broadcastInputIndex = 0;
      for (NamedChannel broadcastInput: node.getBroadcastInputs()) {
View Full Code Here

    int numDynamicSenderTasksTotal = 0;
   

    // expand the channel to all the union channels, in case there is a union operator at its source
    while (allInChannels.hasNext()) {
      final Channel inConn = allInChannels.next();
     
      // sanity check the common serializer
      if (typeSerFact == null) {
        typeSerFact = inConn.getSerializer();
      } else if (!typeSerFact.equals(inConn.getSerializer())) {
        throw new CompilerException("Conflicting types in union operator.");
      }
     
      final PlanNode sourceNode = inConn.getSource();
      AbstractJobVertex sourceVertex = this.vertices.get(sourceNode);
      TaskConfig sourceVertexConfig;

      if (sourceVertex == null) {
        // this predecessor is chained to another task or an iteration
        final TaskInChain chainedTask;
        final IterationDescriptor iteration;
        if ((chainedTask = this.chainedTasks.get(sourceNode)) != null) {
          // push chained task
          if (chainedTask.getContainingVertex() == null) {
            throw new IllegalStateException("Bug: Chained task has not been assigned its containing vertex when connecting.");
          }
          sourceVertex = chainedTask.getContainingVertex();
          sourceVertexConfig = chainedTask.getTaskConfig();
        } else if ((iteration = this.iterations.get(sourceNode)) != null) {
          // predecessor is an iteration
          sourceVertex = iteration.getHeadTask();
          sourceVertexConfig = iteration.getHeadFinalResultConfig();
        } else {
          throw new CompilerException("Bug: Could not resolve source node for a channel.");
        }
      } else {
        // predecessor is its own vertex
        sourceVertexConfig = new TaskConfig(sourceVertex.getConfiguration());
      }
      DistributionPattern pattern = connectJobVertices(
        inConn, inputIndex, sourceVertex, sourceVertexConfig, targetVertex, targetVertexConfig, isBroadcast);
     
      // accounting on channels and senders
      numChannelsTotal++;
      if (inConn.isOnDynamicPath()) {
        numChannelsDynamicPath++;
        numDynamicSenderTasksTotal += getNumberOfSendersPerReceiver(pattern,
          sourceVertex.getNumberOfSubtasks(), targetVertex.getNumberOfSubtasks());
      }
    }
View Full Code Here

    final DriverStrategy ds = node.getDriverStrategy();
   
    // check, whether chaining is possible
    boolean chaining = false;
    {
      Channel inConn = node.getInput();
      PlanNode pred = inConn.getSource();
      chaining = ds.getPushChainDriverClass() != null &&
          !(pred instanceof NAryUnionPlanNode) &&  // first op after union is stand-alone, because union is merged
          !(pred instanceof BulkPartialSolutionPlanNode) &&  // partial solution merges anyways
          !(pred instanceof WorksetPlanNode) &&  // workset merges anyways
          !(pred instanceof IterationPlanNode) && // cannot chain with iteration heads currently
          inConn.getShipStrategy() == ShipStrategyType.FORWARD &&
          inConn.getLocalStrategy() == LocalStrategy.NONE &&
          pred.getOutgoingChannels().size() == 1 &&
          node.getDegreeOfParallelism() == pred.getDegreeOfParallelism() &&
          node.getSubtasksPerInstance() == pred.getSubtasksPerInstance() &&
          node.getBroadcastInputs().isEmpty();
     
View Full Code Here

  @SuppressWarnings("unchecked")
  protected void traverse(PlanNode node, T parentSchema, boolean createUtilities) {
    // distinguish the node types
    if (node instanceof SinkPlanNode) {
      SinkPlanNode sn = (SinkPlanNode) node;
      Channel inchannel = sn.getInput();
     
      T schema = createEmptySchema();
      sn.postPassHelper = schema;
     
      // add the sinks information to the schema
View Full Code Here

    // 5) There is no local strategy on the edge for the initial partial solution, as
    //    this translates to a local strategy that would only be executed in the first iteration
   
    final boolean merge;
    if (mergeIterationAuxTasks && pspn.getOutgoingChannels().size() == 1) {
      final Channel c = pspn.getOutgoingChannels().get(0);
      final PlanNode successor = c.getTarget();
      merge = c.getShipStrategy() == ShipStrategyType.FORWARD &&
          c.getLocalStrategy() == LocalStrategy.NONE &&
          c.getTempMode() == TempMode.NONE &&
          successor.getDegreeOfParallelism() == pspn.getDegreeOfParallelism() &&
          successor.getSubtasksPerInstance() == pspn.getSubtasksPerInstance() &&
          !(successor instanceof NAryUnionPlanNode) &&
          successor != iteration.getRootOfStepFunction() &&
          iteration.getInput().getLocalStrategy() == LocalStrategy.NONE;
View Full Code Here

    // 5) There is no local strategy on the edge for the initial workset, as
    //    this translates to a local strategy that would only be executed in the first superstep
   
    final boolean merge;
    if (mergeIterationAuxTasks && wspn.getOutgoingChannels().size() == 1) {
      final Channel c = wspn.getOutgoingChannels().get(0);
      final PlanNode successor = c.getTarget();
      merge = c.getShipStrategy() == ShipStrategyType.FORWARD &&
          c.getLocalStrategy() == LocalStrategy.NONE &&
          c.getTempMode() == TempMode.NONE &&
          successor.getDegreeOfParallelism() == wspn.getDegreeOfParallelism() &&
          successor.getSubtasksPerInstance() == wspn.getSubtasksPerInstance() &&
          !(successor instanceof NAryUnionPlanNode) &&
          successor != iteration.getNextWorkSetPlanNode() &&
          iteration.getInitialWorksetInput().getLocalStrategy() == LocalStrategy.NONE;
View Full Code Here

    {
      return new SingleInputPlanNode(node, "Reduce ("+node.getPactContract().getName()+")", in, DriverStrategy.SORTED_REDUCE, this.keyList);
    }
    else {
      // non forward case. all local properties are killed anyways, so we can safely plug in a combiner
      Channel toCombiner = new Channel(in.getSource());
      toCombiner.setShipStrategy(ShipStrategyType.FORWARD);
     
      // create an input node for combine with same DOP as input node
      ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
      combinerNode.setDegreeOfParallelism(in.getSource().getDegreeOfParallelism());
      combinerNode.setSubtasksPerInstance(in.getSource().getSubtasksPerInstance());
     
      SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getPactContract().getName()+")", toCombiner, DriverStrategy.SORTED_PARTIAL_REDUCE, this.keyList);
      combiner.setCosts(new Costs(0, 0));
      combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
     
      Channel toReducer = new Channel(combiner);
      toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder());
      toReducer.setLocalStrategy(LocalStrategy.SORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
      return new SingleInputPlanNode(node, "Reduce("+node.getPactContract().getName()+")", toReducer, DriverStrategy.SORTED_REDUCE, this.keyList);
    }
  }
View Full Code Here

TOP

Related Classes of eu.stratosphere.compiler.plan.Channel

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.