Package eu.stratosphere.compiler.plan

Examples of eu.stratosphere.compiler.plan.Channel


            }
          }
         
          // assign memory to the local and global strategies of the channels
          for (Iterator<Channel> channels = node.getInputs(); channels.hasNext();) {
            final Channel c = channels.next();
            if (c.getLocalStrategy().dams()) {
              final long mem = memoryPerInstanceAndWeight / node.getSubtasksPerInstance();
              c.setMemoryLocalStrategy(mem);
              if (LOG.isDebugEnabled()) {
                final long mib = mem >> 20;
                LOG.debug("Assigned " + mib + " MiBytes memory to each local strategy instance of " +
                  c + " (" + mib * node.getDegreeOfParallelism() + " MiBytes total.)");
              }
            }
            if (c.getTempMode() != TempMode.NONE) {
              final long mem = memoryPerInstanceAndWeight / node.getSubtasksPerInstance();
              c.setTempMemory(mem);
              if (LOG.isDebugEnabled()) {
                final long mib = mem >> 20;
                LOG.debug("Assigned " + mib + " MiBytes memory to each instance of the temp table for " +
                  c + " (" + mib * node.getDegreeOfParallelism() + " MiBytes total.)");
              }
View Full Code Here


      }
     
      // double-connect the connections. previously, only parents knew their children, because
      // one child candidate could have been referenced by multiple parents.
      for (Iterator<Channel> iter = visitable.getInputs(); iter.hasNext();) {
        final Channel conn = iter.next();
        conn.setTarget(visitable);
        conn.getSource().addOutgoingChannel(conn);
      }
     
      for (Channel c : visitable.getBroadcastInputs()) {
        c.setTarget(visitable);
        c.getSource().addOutgoingChannel(c);
      }

      // count the memory consumption
      this.memoryConsumerWeights += visitable.getMemoryConsumerWeight();
      for (Iterator<Channel> channels = visitable.getInputs(); channels.hasNext();) {
        final Channel c = channels.next();
        if (c.getLocalStrategy().dams()) {
          this.memoryConsumerWeights++;
        }
        if (c.getTempMode() != TempMode.NONE) {
          this.memoryConsumerWeights++;
        }
      }
      for (Channel c : visitable.getBroadcastInputs()) {
        if (c.getLocalStrategy().dams()) {
          this.memoryConsumerWeights++;
        }
        if (c.getTempMode() != TempMode.NONE) {
          this.memoryConsumerWeights++;
        }
      }
     
      // pass the visitor to the iteraton's step function
View Full Code Here

    @Override
    public void postVisit(PlanNode visitable) {
     
      if (visitable instanceof BinaryUnionPlanNode) {
        final BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
        final Channel in1 = unionNode.getInput1();
        final Channel in2 = unionNode.getInput2();
     
        PlanNode newUnionNode;
       
        // if any input is cached, we keep this as a binary union and do not collapse it into a
        // n-ary union
View Full Code Here

    // create all candidates
    for (PlanNode child : subPlans) {
      if (this.inConn.getShipStrategy() == null) {
        // pick the strategy ourselves
        for (RequestedGlobalProperties igps: intGlobal) {
          final Channel c = new Channel(child, this.inConn.getMaterializationMode());
          igps.parameterizeChannel(c, globalDopChange, localDopChange);
         
          // if the DOP changed, make sure that we cancel out properties, unless the
          // ship strategy preserves/establishes them even under changing DOPs
          if (globalDopChange && !c.getShipStrategy().isNetworkStrategy()) {
            c.getGlobalProperties().reset();
          }
          if (localDopChange && !(c.getShipStrategy().isNetworkStrategy() ||
                c.getShipStrategy().compensatesForLocalDOPChanges())) {
            c.getGlobalProperties().reset();
          }
         
          // check whether we meet any of the accepted properties
          // we may remove this check, when we do a check to not inherit
          // requested global properties that are incompatible with all possible
          // requested properties
          for (RequestedGlobalProperties rgps: allValidGlobals) {
            if (rgps.isMetBy(c.getGlobalProperties())) {
              addLocalCandidates(c, broadcastPlanChannels, igps, outputPlans, estimator);
              break;
            }
          }
        }
      } else {
        // hint fixed the strategy
        final Channel c = new Channel(child, this.inConn.getMaterializationMode());
        if (this.keys != null) {
          c.setShipStrategy(this.inConn.getShipStrategy(), this.keys.toFieldList());
        } else {
          c.setShipStrategy(this.inConn.getShipStrategy());
        }
       
        if (globalDopChange) {
          c.adjustGlobalPropertiesForFullParallelismChange();
        } else if (localDopChange) {
          c.adjustGlobalPropertiesForLocalParallelismChange();
        }
       
        // check whether we meet any of the accepted properties
        for (RequestedGlobalProperties rgps: allValidGlobals) {
          if (rgps.isMetBy(c.getGlobalProperties())) {
            addLocalCandidates(c, broadcastPlanChannels, rgps, outputPlans, estimator);
            break;
          }
        }
      }
View Full Code Here

  protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
      List<PlanNode> target, CostEstimator estimator)
  {
    final LocalProperties lp = template.getLocalPropertiesAfterShippingOnly();
    for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
      final Channel in = template.clone();
      if (ilp.isMetBy(lp)) {
        in.setLocalStrategy(LocalStrategy.NONE);
      } else {
        ilp.parameterizeChannel(in);
      }
     
      // instantiate a candidate, if the instantiated local properties meet one possible local property set
      for (OperatorDescriptorSingle dps: getPossibleProperties()) {
        for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
          if (ilps.isMetBy(in.getLocalProperties())) {
            instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
            break;
          }
        }
      }
View Full Code Here

          // output connection side
          if (inConns.hasNext() || inputNum > 0) {
            writer.print(", \"side\": \"" + (inputNum == 0 ? "first" : "second") + "\"");
          }
          // output shipping strategy and channel type
          final Channel channel = (inConn instanceof Channel) ? (Channel) inConn : null;
          final ShipStrategyType shipType = channel != null ? channel.getShipStrategy() :
              ((PactConnection) inConn).getShipStrategy();
           
          String shipStrategy = null;
          if (shipType != null) {
            switch (shipType) {
            case NONE:
              // nothing
              break;
            case FORWARD:
              shipStrategy = "Forward";
              break;
            case BROADCAST:
              shipStrategy = "Broadcast";
              break;
            case PARTITION_HASH:
              shipStrategy = "Hash Partition";
              break;
            case PARTITION_RANGE:
              shipStrategy = "Range Partition";
              break;
            case PARTITION_LOCAL_HASH:
              shipStrategy = "Hash Partition (local)";
              break;
            case PARTITION_RANDOM:
              shipStrategy = "Redistribute";
              break;
            default:
              throw new CompilerException("Unknown ship strategy '" + conn.getShipStrategy().name()
                + "' in JSON generator.");
            }
          }
         
          if (channel != null && channel.getShipStrategyKeys() != null && channel.getShipStrategyKeys().size() > 0) {
            shipStrategy += " on " + (channel.getShipStrategySortOrder() == null ?
                channel.getShipStrategyKeys().toString() :
                Utils.createOrdering(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()).toString());
          }
 
          if (shipStrategy != null) {
            writer.print(", \"ship_strategy\": \"" + shipStrategy + "\"");
          }
         
          if (channel != null) {
            String localStrategy = null;
            switch (channel.getLocalStrategy()) {
            case NONE:
              break;
            case SORT:
              localStrategy = "Sort";
              break;
            case COMBININGSORT:
              localStrategy = "Sort (combining)";
              break;
            default:
              throw new CompilerException("Unknown local strategy " + channel.getLocalStrategy().name());
            }
           
            if (channel != null && channel.getLocalStrategyKeys() != null && channel.getLocalStrategyKeys().size() > 0) {
              localStrategy += " on " + (channel.getLocalStrategySortOrder() == null ?
                  channel.getLocalStrategyKeys().toString() :
                  Utils.createOrdering(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder()).toString());
            }
           
            if (localStrategy != null) {
              writer.print(", \"local_strategy\": \"" + localStrategy + "\"");
            }
           
            if (channel != null && channel.getTempMode() != TempMode.NONE) {
              String tempMode = channel.getTempMode().toString();
              writer.print(", \"temp_mode\": \"" + tempMode + "\"");
            }
          }
         
          writer.print('}');
View Full Code Here

   
    // distinguish the node types
    if (node instanceof SinkPlanNode) {
      // descend to the input channel
      SinkPlanNode sn = (SinkPlanNode) node;
      Channel inchannel = sn.getInput();
      traverseChannel(inchannel);
    }
    else if (node instanceof SourcePlanNode) {
      TypeInformation<?> typeInfo = getTypeInfoFromSource((SourcePlanNode) node);
      ((SourcePlanNode) node).setSerializer(createSerializer(typeInfo));
View Full Code Here

    if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
      // locally connected, directly instantiate
      return new SingleInputPlanNode(node, "GroupReduce ("+node.getPactContract().getName()+")", in, DriverStrategy.ALL_GROUP_REDUCE);
    } else {
      // non forward case.plug in a combiner
      Channel toCombiner = new Channel(in.getSource());
      toCombiner.setShipStrategy(ShipStrategyType.FORWARD);
     
      // create an input node for combine with same DOP as input node
      GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
      combinerNode.setDegreeOfParallelism(in.getSource().getDegreeOfParallelism());
      combinerNode.setSubtasksPerInstance(in.getSource().getSubtasksPerInstance());
     
      SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getPactContract().getName()+")", toCombiner, DriverStrategy.ALL_GROUP_COMBINE);
      combiner.setCosts(new Costs(0, 0));
      combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
     
      Channel toReducer = new Channel(combiner);
      toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder());
      toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
      return new SingleInputPlanNode(node, "GroupReduce ("+node.getPactContract().getName()+")", toReducer, DriverStrategy.ALL_GROUP_REDUCE);
    }
  }
View Full Code Here

          continue;
        }
       
        for (RequestedGlobalProperties igps: this.channelProps) {
          // create a candidate channel for the first input. mark it cached, if the connection says so
          Channel c1 = new Channel(child1, this.input1.getMaterializationMode());
          if (this.input1.getShipStrategy() == null) {
            // free to choose the ship strategy
            igps.parameterizeChannel(c1, globalDopChange1, localDopChange1);
           
            // if the DOP changed, make sure that we cancel out properties, unless the
            // ship strategy preserves/establishes them even under changing DOPs
            if (globalDopChange1 && !c1.getShipStrategy().isNetworkStrategy()) {
              c1.getGlobalProperties().reset();
            }
            if (localDopChange1 && !(c1.getShipStrategy().isNetworkStrategy() ||
                  c1.getShipStrategy().compensatesForLocalDOPChanges())) {
              c1.getGlobalProperties().reset();
            }
          } else {
            // ship strategy fixed by compiler hint
            if (this.keys1 != null) {
              c1.setShipStrategy(this.input1.getShipStrategy(), this.keys1.toFieldList());
            } else {
              c1.setShipStrategy(this.input1.getShipStrategy());
            }
           
            if (globalDopChange1) {
              c1.adjustGlobalPropertiesForFullParallelismChange();
            } else if (localDopChange1) {
              c1.adjustGlobalPropertiesForLocalParallelismChange();
            }
          }
         
          // create a candidate channel for the first input. mark it cached, if the connection says so
          Channel c2 = new Channel(child2, this.input2.getMaterializationMode());
          if (this.input2.getShipStrategy() == null) {
            // free to choose the ship strategy
            igps.parameterizeChannel(c2, globalDopChange2, localDopChange2);
           
            // if the DOP changed, make sure that we cancel out properties, unless the
            // ship strategy preserves/establishes them even under changing DOPs
            if (globalDopChange2 && !c2.getShipStrategy().isNetworkStrategy()) {
              c2.getGlobalProperties().reset();
            }
            if (localDopChange2 && !(c2.getShipStrategy().isNetworkStrategy() ||
                  c2.getShipStrategy().compensatesForLocalDOPChanges())) {
              c2.getGlobalProperties().reset();
            }
          } else {
            // ship strategy fixed by compiler hint
            if (this.keys2 != null) {
              c2.setShipStrategy(this.input2.getShipStrategy(), this.keys2.toFieldList());
            } else {
              c2.setShipStrategy(this.input2.getShipStrategy());
            }
           
            if (globalDopChange2) {
              c2.adjustGlobalPropertiesForFullParallelismChange();
            } else if (localDopChange2) {
              c2.adjustGlobalPropertiesForLocalParallelismChange();
            }
          }
         
          // get the global properties and clear unique fields (not preserved anyways during the union)
          GlobalProperties p1 = c1.getGlobalProperties();
          GlobalProperties p2 = c2.getGlobalProperties();
          p1.clearUniqueFieldCombinations();
          p2.clearUniqueFieldCombinations();
         
          // adjust the partitionings, if they exist but are not equal. this may happen when both channels have a
          // partitioning that fulfills the requirements, but both are incompatible. For example may a property requirement
          // be ANY_PARTITIONING on fields (0) and one channel is range partitioned on that field, the other is hash
          // partitioned on that field.
          if (!igps.isTrivial() && !(p1.equals(p2))) {
            if (c1.getShipStrategy() == ShipStrategyType.FORWARD && c2.getShipStrategy() != ShipStrategyType.FORWARD) {
              // adjust c2 to c1
              c2 = c2.clone();
              p1.parameterizeChannel(c2,globalDopChange2);
            } else if (c2.getShipStrategy() == ShipStrategyType.FORWARD && c1.getShipStrategy() != ShipStrategyType.FORWARD) {
              // adjust c1 to c2
              c1 = c1.clone();
              p2.parameterizeChannel(c1,globalDopChange1);
            } else if (c1.getShipStrategy() == ShipStrategyType.FORWARD && c2.getShipStrategy() == ShipStrategyType.FORWARD) {
              boolean adjustC1 = c1.getEstimatedOutputSize() <= 0 || c2.getEstimatedOutputSize() <= 0 ||
                  c1.getEstimatedOutputSize() <= c2.getEstimatedOutputSize();
              if (adjustC1) {
                c2 = c2.clone();
                p1.parameterizeChannel(c2, globalDopChange2);
              } else {
                c1 = c1.clone();
                p2.parameterizeChannel(c1, globalDopChange1);
              }
View Full Code Here

    if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
      // locally connected, directly instantiate
      return new SingleInputPlanNode(node, "Reduce ("+node.getPactContract().getName()+")", in, DriverStrategy.ALL_REDUCE);
    } else {
      // non forward case.plug in a combiner
      Channel toCombiner = new Channel(in.getSource());
      toCombiner.setShipStrategy(ShipStrategyType.FORWARD);
     
      // create an input node for combine with same DOP as input node
      ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
      combinerNode.setDegreeOfParallelism(in.getSource().getDegreeOfParallelism());
      combinerNode.setSubtasksPerInstance(in.getSource().getSubtasksPerInstance());
     
      SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getPactContract().getName()+")", toCombiner, DriverStrategy.ALL_REDUCE);
      combiner.setCosts(new Costs(0, 0));
      combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
     
      Channel toReducer = new Channel(combiner);
      toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder());
      toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
      return new SingleInputPlanNode(node, "Reduce("+node.getPactContract().getName()+")", toReducer, DriverStrategy.ALL_REDUCE);
    }
  }
View Full Code Here

TOP

Related Classes of eu.stratosphere.compiler.plan.Channel

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.