Package eu.stratosphere.compiler.plan

Examples of eu.stratosphere.compiler.plan.PlanNode


  }

  protected void instantiateCandidate(OperatorDescriptorSingle dps, Channel in, List<Set<? extends NamedChannel>> broadcastPlanChannels,
      List<PlanNode> target, CostEstimator estimator, RequestedGlobalProperties globPropsReq, RequestedLocalProperties locPropsReq)
  {
    final PlanNode inputSource = in.getSource();
   
    for (List<NamedChannel> broadcastChannelsCombination: Sets.cartesianProduct(broadcastPlanChannels)) {
     
      boolean validCombination = true;
     
      // check whether the broadcast inputs use the same plan candidate at the branching point
      for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
        NamedChannel nc = broadcastChannelsCombination.get(i);
        PlanNode bcSource = nc.getSource();
       
        // check branch compatibility against input
        if (!areBranchCompatible(bcSource, inputSource)) {
          validCombination = false;
          break;
        }
       
        // check branch compatibility against all other broadcast variables
        for (int k = 0; k < i; k++) {
          PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
         
          if (!areBranchCompatible(bcSource, otherBcSource)) {
            validCombination = false;
            break;
          }
View Full Code Here


    List<PlanNode> candidates = this.nextPartialSolution.getAlternativePlans(estimator);
   
    // 4) Throw away all that are not compatible with the properties currently requested to the
    //    initial partial solution
    for (Iterator<PlanNode> planDeleter = candidates.iterator(); planDeleter.hasNext(); ) {
      PlanNode candidate = planDeleter.next();
      if (!(globPropsReq.isMetBy(candidate.getGlobalProperties()) && locPropsReq.isMetBy(candidate.getLocalProperties()))) {
        planDeleter.remove();
      }
    }
   
    // 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
    if (terminationCriterion == null) {
      for (PlanNode candidate : candidates) {
        BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getPactContract().getName()+")", in, pspn, candidate);
        GlobalProperties gProps = candidate.getGlobalProperties().clone();
        LocalProperties lProps = candidate.getLocalProperties().clone();
        node.initProperties(gProps, lProps);
        target.add(node);
      }
    }
    else if(candidates.size() > 0) {
      List<PlanNode> terminationCriterionCandidates = this.terminationCriterion.getAlternativePlans(estimator);

      SingleRootJoiner singleRoot = (SingleRootJoiner) this.singleRoot;
     
      for (PlanNode candidate : candidates) {
        for(PlanNode terminationCandidate : terminationCriterionCandidates) {
          if (singleRoot.areBranchCompatible(candidate, terminationCandidate)) {
            BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getPactContract().getName()+")", in, pspn, candidate, terminationCandidate);
            GlobalProperties gProps = candidate.getGlobalProperties().clone();
            LocalProperties lProps = candidate.getLocalProperties().clone();
            node.initProperties(gProps, lProps);
            target.add(node);
           
          }
        }
View Full Code Here

      Comparator<PlanNode> sorter = new Comparator<PlanNode>() {
       
        @Override
        public int compare(PlanNode o1, PlanNode o2) {
          for (int i = 0; i < branchDeterminers.length; i++) {
            PlanNode n1 = o1.getCandidateAtBranchPoint(branchDeterminers[i]);
            PlanNode n2 = o2.getCandidateAtBranchPoint(branchDeterminers[i]);
            int hash1 = System.identityHashCode(n1);
            int hash2 = System.identityHashCode(n2);
           
            if (hash1 != hash2) {
              return hash1 - hash2;
            }
          }
          return 0;
        }
      };
      Collections.sort(plans, sorter);
     
      List<PlanNode> result = new ArrayList<PlanNode>();
      List<PlanNode> turn = new ArrayList<PlanNode>();
     
      final PlanNode[] determinerChoice = new PlanNode[branchDeterminers.length];

      while (!plans.isEmpty()) {
        // take one as the determiner
        turn.clear();
        PlanNode determiner = plans.remove(plans.size() - 1);
        turn.add(determiner);
       
        for (int i = 0; i < determinerChoice.length; i++) {
          determinerChoice[i] = determiner.getCandidateAtBranchPoint(branchDeterminers[i]);
        }

        // go backwards through the plans and find all that are equal
        boolean stillEqual = true;
        for (int k = plans.size() - 1; k >= 0 && stillEqual; k--) {
          PlanNode toCheck = plans.get(k);
         
          for (int i = 0; i < branchDeterminers.length; i++) {
            PlanNode checkerChoice = toCheck.getCandidateAtBranchPoint(branchDeterminers[i]);
         
            if (checkerChoice != determinerChoice[i]) {
              // not the same anymore
              stillEqual = false;
              break;
View Full Code Here

   
    final PlanNode[][] toKeep = new PlanNode[gps.length][];
    final PlanNode[] cheapestForGlobal = new PlanNode[gps.length];
   
   
    PlanNode cheapest = null; // the overall cheapest plan

    // go over all plans from the list
    for (PlanNode candidate : plans) {
      // check if that plan is the overall cheapest
      if (cheapest == null || (cheapest.getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0)) {
        cheapest = candidate;
      }

      // find the interesting global properties that this plan matches
      for (int i = 0; i < gps.length; i++) {
        if (gps[i].isMetBy(candidate.getGlobalProperties())) {
          // the candidate meets the global property requirements. That means
          // it has a chance that its local properties are re-used (they would be
          // destroyed if global properties need to be established)
         
          if (cheapestForGlobal[i] == null || (cheapestForGlobal[i].getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0)) {
            cheapestForGlobal[i] = candidate;
          }
         
          final PlanNode[] localMatches;
          if (toKeep[i] == null) {
            localMatches = new PlanNode[lps.length];
            toKeep[i] = localMatches;
          } else {
            localMatches = toKeep[i];
          }
         
          for (int k = 0; k < lps.length; k++) {
            if (lps[k].isMetBy(candidate.getLocalProperties())) {
              final PlanNode previous = localMatches[k];
              if (previous == null || previous.getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0) {
                // this one is cheaper!
                localMatches[k] = candidate;
              }
            }
          }
        }
      }
    }

    // all plans are set now
    plans.clear();

    // add the cheapest plan
    if (cheapest != null) {
      plans.add(cheapest);
      cheapest.setPruningMarker(); // remember that that plan is in the set
    }
   
    // skip the top down delta cost check for now (TODO: implement this)
    // add all others, which are optimal for some interesting properties
    for (int i = 0; i < gps.length; i++) {
      if (toKeep[i] != null) {
        final PlanNode[] localMatches = toKeep[i];
        for (int k = 0; k < localMatches.length; k++) {
          final PlanNode n = localMatches[k];
          if (n != null && !n.isPruneMarkerSet()) {
            n.setPruningMarker();
            plans.add(n);
          }
        }
      }
      if (cheapestForGlobal[i] != null) {
        final PlanNode n = cheapestForGlobal[i];
        if (!n.isPruneMarkerSet()) {
          n.setPruningMarker();
          plans.add(n);
        }
      }
    }
  }
View Full Code Here

    if (this.hereJoinedBranches == null || this.hereJoinedBranches.isEmpty()) {
      return true;
    }

    for (OptimizerNode joinedBrancher : hereJoinedBranches) {
      final PlanNode branch1Cand = plan1.getCandidateAtBranchPoint(joinedBrancher);
      final PlanNode branch2Cand = plan2.getCandidateAtBranchPoint(joinedBrancher);
     
      if (branch1Cand != null && branch2Cand != null && branch1Cand != branch2Cand) {
        return false;
      }
    }
View Full Code Here

      if (visitable instanceof BinaryUnionPlanNode) {
        final BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
        final Channel in1 = unionNode.getInput1();
        final Channel in2 = unionNode.getInput2();
     
        PlanNode newUnionNode;
       
        // if any input is cached, we keep this as a binary union and do not collapse it into a
        // n-ary union
//        if (in1.getTempMode().isCached() || in2.getTempMode().isCached()) {
//          // replace this node by an explicit operator
View Full Code Here

   
    //---------------------------------------------------------------------------------------
    // the part below here is relevant only to plan nodes with concrete strategies, etc
    //---------------------------------------------------------------------------------------

    final PlanNode p = node.getPlanNode();
    if (p == null) {
      // finish node
      writer.print("\n\t}");
      return;
    }
    // local strategy
    String locString = null;
    if (p.getDriverStrategy() != null) {
      switch (p.getDriverStrategy()) {
      case NONE:
      case BINARY_NO_OP:
        break;
       
      case UNARY_NO_OP:
        locString = "No-Op";
        break;
       
      case COLLECTOR_MAP:
      case MAP:
      case FLAT_MAP:
        locString = "Map";
        break;
     
      case ALL_REDUCE:
        locString = "Reduce All";
        break;
     
      case ALL_GROUP_REDUCE:
      case ALL_GROUP_COMBINE:
        locString = "Group Reduce All";
        break;
       
      case SORTED_REDUCE:
        locString = "Sorted Reduce";
        break;
       
      case SORTED_PARTIAL_REDUCE:
        locString = "Sorted Combine/Reduce";
        break;

      case SORTED_GROUP_REDUCE:
        locString = "Sorted Group Reduce";
        break;
       
      case SORTED_GROUP_COMBINE:
        locString = "Sorted Combine";
        break;

      case HYBRIDHASH_BUILD_FIRST:
        locString = "Hybrid Hash (build: " + child1name + ")";
        break;
      case HYBRIDHASH_BUILD_SECOND:
        locString = "Hybrid Hash (build: " + child2name + ")";
        break;

      case NESTEDLOOP_BLOCKED_OUTER_FIRST:
        locString = "Nested Loops (Blocked Outer: " + child1name + ")";
        break;
      case NESTEDLOOP_BLOCKED_OUTER_SECOND:
        locString = "Nested Loops (Blocked Outer: " + child2name + ")";
        break;
      case NESTEDLOOP_STREAMED_OUTER_FIRST:
        locString = "Nested Loops (Streamed Outer: " + child1name + ")";
        break;
      case NESTEDLOOP_STREAMED_OUTER_SECOND:
        locString = "Nested Loops (Streamed Outer: " + child2name + ")";
        break;

      case MERGE:
        locString = "Merge";
        break;

      case CO_GROUP:
        locString = "Co-Group";
        break;

      default:
        throw new CompilerException("Unknown local strategy '" + p.getDriverStrategy().name()
          + "' in JSON generator.");
      }

      if (locString != null) {
        writer.print(",\n\t\t\"driver_strategy\": \"");
        writer.print(locString);
        writer.print("\"");
      }
    }
   
    {
      // output node global properties
      final GlobalProperties gp = p.getGlobalProperties();

      writer.print(",\n\t\t\"global_properties\": [\n");

      addProperty(writer, "Partitioning", gp.getPartitioning().name(), true);
      if (gp.getPartitioningFields() != null) {
        addProperty(writer, "Partitioned on", gp.getPartitioningFields().toString(), false);
      }
      if (gp.getPartitioningOrdering() != null) {
        addProperty(writer, "Partitioning Order", gp.getPartitioningOrdering().toString(), false)
      }
      else {
        addProperty(writer, "Partitioning Order", "(none)", false);
      }
      if (n.getUniqueFields() == null || n.getUniqueFields().size() == 0) {
        addProperty(writer, "Uniqueness", "not unique", false);
      }
      else {
        addProperty(writer, "Uniqueness", n.getUniqueFields().toString(), false)
      }

      writer.print("\n\t\t]");
    }

    {
      // output node local properties
      LocalProperties lp = p.getLocalProperties();

      writer.print(",\n\t\t\"local_properties\": [\n");

      if (lp.getOrdering() != null) {
        addProperty(writer, "Order", lp.getOrdering().toString(), true)
      }
      else {
        addProperty(writer, "Order", "(none)", true);
      }
      if (lp.getGroupedFields() != null && lp.getGroupedFields().size() > 0) {
        addProperty(writer, "Grouped on", lp.getGroupedFields().toString(), false);
      } else {
        addProperty(writer, "Grouping", "not grouped", false)
      }
      if (n.getUniqueFields() == null || n.getUniqueFields().size() == 0) {
        addProperty(writer, "Uniqueness", "not unique", false);
      }
      else {
        addProperty(writer, "Uniqueness", n.getUniqueFields().toString(), false)
      }

      writer.print("\n\t\t]");
    }

    // output node size estimates
    writer.print(",\n\t\t\"estimates\": [\n");

    addProperty(writer, "Est. Output Size", n.getEstimatedOutputSize() == -1 ? "(unknown)"
      : formatNumber(n.getEstimatedOutputSize(), "B"), true);
    addProperty(writer, "Est. Cardinality", n.getEstimatedNumRecords() == -1 ? "(unknown)"
      : formatNumber(n.getEstimatedNumRecords()), false);

    writer.print("\t\t]");

    // output node cost
    if (p.getNodeCosts() != null) {
      writer.print(",\n\t\t\"costs\": [\n");

      addProperty(writer, "Network", p.getNodeCosts().getNetworkCost() == -1 ? "(unknown)"
        : formatNumber(p.getNodeCosts().getNetworkCost(), "B"), true);
      addProperty(writer, "Disk I/O", p.getNodeCosts().getDiskCost() == -1 ? "(unknown)"
        : formatNumber(p.getNodeCosts().getDiskCost(), "B"), false);
      addProperty(writer, "CPU", p.getNodeCosts().getCpuCost() == -1 ? "(unknown)"
        : formatNumber(p.getNodeCosts().getCpuCost(), ""), false);

      addProperty(writer, "Cumulative Network",
        p.getCumulativeCosts().getNetworkCost() == -1 ? "(unknown)" : formatNumber(p
          .getCumulativeCosts().getNetworkCost(), "B"), false);
      addProperty(writer, "Cumulative Disk I/O",
        p.getCumulativeCosts().getDiskCost() == -1 ? "(unknown)" : formatNumber(p
          .getCumulativeCosts().getDiskCost(), "B"), false);
      addProperty(writer, "Cumulative CPU",
        p.getCumulativeCosts().getCpuCost() == -1 ? "(unknown)" : formatNumber(p
          .getCumulativeCosts().getCpuCost(), ""), false);

      writer.print("\n\t\t]");
    }
View Full Code Here

    }
  }
 
  private void traverseChannel(Channel channel) {
   
    PlanNode source = channel.getSource();
    Operator<?> javaOp = source.getPactContract();
   
//    if (!(javaOp instanceof BulkIteration) && !(javaOp instanceof JavaPlanNode)) {
//      throw new RuntimeException("Wrong operator type found in post pass: " + javaOp);
//    }

    TypeInformation<?> type = javaOp.getOperatorInfo().getOutputType();


    if(javaOp instanceof GroupReduceOperatorBase &&
        (source.getDriverStrategy() == DriverStrategy.SORTED_GROUP_COMBINE || source.getDriverStrategy() == DriverStrategy.ALL_GROUP_COMBINE)) {
      GroupReduceOperatorBase<?, ?, ?> groupNode = (GroupReduceOperatorBase<?, ?, ?>) javaOp;
      type = groupNode.getInput().getOperatorInfo().getOutputType();
    }
    else if(javaOp instanceof PlanUnwrappingReduceGroupOperator &&
        source.getDriverStrategy().equals(DriverStrategy.SORTED_GROUP_COMBINE)) {
      PlanUnwrappingReduceGroupOperator<?, ?, ?> groupNode = (PlanUnwrappingReduceGroupOperator<?, ?, ?>) javaOp;
      type = groupNode.getInput().getOperatorInfo().getOutputType();
    }
   
    // the serializer always exists
View Full Code Here

      throw new CompilerException("Error in compiler: more than one best plan was created!");
    }

    // check if the best plan's root is a data sink (single sink plan)
    // if so, directly take it. if it is a sink joiner node, get its contained sinks
    PlanNode bestPlanRoot = bestPlan.get(0);
    List<SinkPlanNode> bestPlanSinks = new ArrayList<SinkPlanNode>(4);

    if (bestPlanRoot instanceof SinkPlanNode) {
      bestPlanSinks.add((SinkPlanNode) bestPlanRoot);
    } else if (bestPlanRoot instanceof SinkJoinerPlanNode) {
View Full Code Here

  protected void instantiate(OperatorDescriptorDual operator, Channel in1, Channel in2,
      List<Set<? extends NamedChannel>> broadcastPlanChannels, List<PlanNode> target, CostEstimator estimator,
      RequestedGlobalProperties globPropsReq1, RequestedGlobalProperties globPropsReq2,
      RequestedLocalProperties locPropsReq1, RequestedLocalProperties locPropsReq2)
  {
    final PlanNode inputSource1 = in1.getSource();
    final PlanNode inputSource2 = in2.getSource();
   
    for (List<NamedChannel> broadcastChannelsCombination: Sets.cartesianProduct(broadcastPlanChannels)) {
     
      boolean validCombination = true;
     
      // check whether the broadcast inputs use the same plan candidate at the branching point
      for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
        NamedChannel nc = broadcastChannelsCombination.get(i);
        PlanNode bcSource = nc.getSource();
       
        if (!(areBranchCompatible(bcSource, inputSource1) || areBranchCompatible(bcSource, inputSource2))) {
          validCombination = false;
          break;
        }
       
        // check branch compatibility against all other broadcast variables
        for (int k = 0; k < i; k++) {
          PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
         
          if (!areBranchCompatible(bcSource, otherBcSource)) {
            validCombination = false;
            break;
          }
View Full Code Here

TOP

Related Classes of eu.stratosphere.compiler.plan.PlanNode

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.