Package eu.stratosphere.compiler.plan

Examples of eu.stratosphere.compiler.plan.DualInputPlanNode


      for (Channel c: sn.getBroadcastInputs()) {
        traverseChannel(c);
      }
    }
    else if (node instanceof DualInputPlanNode) {
      DualInputPlanNode dn = (DualInputPlanNode) node;
     
      if (!(dn.getOptimizerNode().getPactContract() instanceof DualInputOperator)) {
        throw new RuntimeException("Wrong operator type found in post pass.");
      }
     
      DualInputOperator<?, ?, ?, ?> dualInputOperator = (DualInputOperator<?, ?, ?, ?>) dn.getOptimizerNode().getPactContract();
     
      // parameterize the node's driver strategy
      if (dn.getDriverStrategy().requiresComparator()) {
        dn.setComparator1(createComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dn.getKeysForInput1(),
          getSortOrders(dn.getKeysForInput1(), dn.getSortOrders())));
        dn.setComparator2(createComparator(dualInputOperator.getOperatorInfo().getSecondInputType(), dn.getKeysForInput2(),
            getSortOrders(dn.getKeysForInput2(), dn.getSortOrders())));

        dn.setPairComparator(createPairComparator(dualInputOperator.getOperatorInfo().getFirstInputType(),
            dualInputOperator.getOperatorInfo().getSecondInputType()));
       
      }
           
      traverseChannel(dn.getInput1());
      traverseChannel(dn.getInput2());
     
      // don't forget the broadcast inputs
      for (Channel c: dn.getBroadcastInputs()) {
        traverseChannel(c);
      }
     
    }
    // catch the sources of the iterative step functions
View Full Code Here


        continue;
      }
     
      placePipelineBreakersIfNecessary(operator.getStrategy(), in1, in2);
     
      DualInputPlanNode node = operator.instantiate(in1, in2, this);
      node.setBroadcastInputs(broadcastChannelsCombination);
     
      GlobalProperties gp1 = in1.getGlobalProperties().clone().filterByNodesConstantSet(this, 0);
      GlobalProperties gp2 = in2.getGlobalProperties().clone().filterByNodesConstantSet(this, 1);
      GlobalProperties combined = operator.computeGlobalProperties(gp1, gp2);

      LocalProperties lp1 = in1.getLocalProperties().clone().filterByNodesConstantSet(this, 0);
      LocalProperties lp2 = in2.getLocalProperties().clone().filterByNodesConstantSet(this, 1);
      LocalProperties locals = operator.computeLocalProperties(lp1, lp2);
     
      node.initProperties(combined, locals);
      node.updatePropertiesWithUniqueSets(getUniqueFields());
      target.add(node);
    }
  }
View Full Code Here

      boolean[] tmp = new boolean[this.keys1.size()];
      System.arraycopy(inputOrders, 0, tmp, 0, tmp.length);
      inputOrders = tmp;
    }
   
    return new DualInputPlanNode(node, "CoGroup ("+node.getPactContract().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders);
  }
View Full Code Here

    return true;
  }

  @Override
  public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
    return new DualInputPlanNode(node, "Join("+node.getPactContract().getName()+")", in1, in2, DriverStrategy.HYBRIDHASH_BUILD_FIRST, this.keys1, this.keys2);
  }
View Full Code Here

            optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
        }
      }
    }
    else if (node instanceof DualInputPlanNode) {
      DualInputPlanNode dn = (DualInputPlanNode) node;
     
      // get the nodes current schema
      T schema1;
      T schema2;
      if (dn.postPassHelper1 == null) {
        schema1 = createEmptySchema();
        schema2 = createEmptySchema();
        dn.postPassHelper1 = schema1;
        dn.postPassHelper2 = schema2;
      } else {
        schema1 = (T) dn.postPassHelper1;
        schema2 = (T) dn.postPassHelper2;
      }

      schema1.increaseNumConnectionsThatContributed();
      schema2.increaseNumConnectionsThatContributed();
      TwoInputNode optNode = dn.getTwoInputNode();
     
      // add the parent schema to the schema
      if (propagateParentSchemaDown) {
        addSchemaToSchema(parentSchema, schema1, optNode, 0);
        addSchemaToSchema(parentSchema, schema2, optNode, 1);
      }
     
      // check whether all outgoing channels have not yet contributed. come back later if not.
      if (schema1.getNumConnectionsThatContributed() < dn.getOutgoingChannels().size()) {
        return;
      }
     
      // add the nodes local information
      try {
        getDualInputNodeSchema(dn, schema1, schema2);
      } catch (ConflictingFieldTypeInfoException e) {
        throw new CompilerPostPassException(getConflictingTypeErrorMessage(e, optNode.getPactContract().getName()));
      }
     
      // parameterize the node's driver strategy
      if (createUtilities) {
        if (dn.getDriverStrategy().requiresComparator()) {
          // set the individual comparators
          try {
            dn.setComparator1(createComparator(dn.getKeysForInput1(), dn.getSortOrders(), schema1));
            dn.setComparator2(createComparator(dn.getKeysForInput2(), dn.getSortOrders(), schema2));
          } catch (MissingFieldTypeInfoException e) {
            throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
                optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
          }
         
          // set the pair comparator
          try {
            dn.setPairComparator(createPairComparator(dn.getKeysForInput1(), dn.getKeysForInput2(),
              dn.getSortOrders(), schema1, schema2));
          } catch (MissingFieldTypeInfoException e) {
            throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
                optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
          }
         
        }
      }
     
      // done, we can now propagate our info down
      try {
        propagateToChannel(schema1, dn.getInput1(), createUtilities);
      } catch (MissingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Could not set up runtime strategy for the first input channel to node '"
          + optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
      }
      try {
        propagateToChannel(schema2, dn.getInput2(), createUtilities);
      } catch (MissingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Could not set up runtime strategy for the second input channel to node '"
          + optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
      }
     
      // don't forget the broadcast inputs
      for (Channel c: dn.getBroadcastInputs()) {
        try {
          propagateToChannel(createEmptySchema(), c, createUtilities);
        } catch (MissingFieldTypeInfoException e) {
          throw new CompilerPostPassException("Could not set up runtime strategy for broadcast channel in node '" +
            optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
View Full Code Here

        // this represents an access into the solution set index.
        // we do not create a vertex for the solution set here (we create the head at the workset place holder)
       
        // we adjust the joins / cogroups that go into the solution set here
        for (Channel c : node.getOutgoingChannels()) {
          DualInputPlanNode target = (DualInputPlanNode) c.getTarget();
          AbstractJobVertex accessingVertex = this.vertices.get(target);
          TaskConfig conf = new TaskConfig(accessingVertex.getConfiguration());
          int inputNum = c == target.getInput1() ? 0 : c == target.getInput2() ? 1 : -1;
         
          // sanity checks
          if (inputNum == -1) {
            throw new CompilerException();
          }
View Full Code Here

      }
     
      // switching build and probe side did not help -> pipeline breaker
      for(DeadlockVertex v : g.vertices) {
        if(v.getOriginal() instanceof DualInputPlanNode) {
          DualInputPlanNode n = (DualInputPlanNode) v.getOriginal();
         
          // what is the pipelined side? (other side should be a dam, otherwise operator could not be source of deadlock)
          if(!(n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput1().getLocalStrategy().dams() || n.getInput1().getTempMode().breaksPipeline())
              && (n.getDriverStrategy().secondDam().equals(DamBehavior.FULL_DAM) || n.getInput2().getLocalStrategy().dams() || n.getInput2().getTempMode().breaksPipeline())) {
            n.getInput1().setTempMode(n.getInput1().getTempMode().makePipelineBreaker());
          }
          else if( !(n.getDriverStrategy().secondDam().equals(DamBehavior.FULL_DAM) || n.getInput2().getLocalStrategy().dams() || n.getInput2().getTempMode().breaksPipeline())
              && (n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput1().getLocalStrategy().dams() || n.getInput1().getTempMode().breaksPipeline())) {
            n.getInput2().setTempMode(n.getInput2().getTempMode().makePipelineBreaker());
          }
         
          // Deadlock resolved?
          if(!hasDeadlock(sinks)) {
            break;
View Full Code Here

      else {
        g.addEdge(n.getPredecessor(), n);
      }
    }
    else if(visitable instanceof DualInputPlanNode) {
      DualInputPlanNode n = (DualInputPlanNode) visitable;
     
      if(n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput1().getLocalStrategy().dams() || n.getInput1().getTempMode().breaksPipeline()) {
        g.addEdge(n, n.getInput1().getSource());
      }
      else {
        g.addEdge(n.getInput1().getSource(), n);
      }
     
      if(!n.getDriverStrategy().equals(DriverStrategy.NONE) && (n.getDriverStrategy().secondDam().equals(DamBehavior.FULL_DAM) || n.getInput2().getLocalStrategy().dams() || n.getInput2().getTempMode().breaksPipeline())) {
        g.addEdge(n, n.getInput2().getSource());
      }
      else {
        g.addEdge(n.getInput2().getSource(), n);
      }
    }
   
   
    // recursively fix iterations
View Full Code Here

    return true;
  }

  @Override
  public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
    return new DualInputPlanNode(node, "Join("+node.getPactContract().getName()+")", in1, in2, DriverStrategy.HYBRIDHASH_BUILD_SECOND, this.keys1, this.keys2);
  }
View Full Code Here

      boolean[] tmp = new boolean[this.keys1.size()];
      System.arraycopy(inputOrders, 0, tmp, 0, tmp.length);
      inputOrders = tmp;
    }
   
    return new DualInputPlanNode(node, "Join("+node.getPactContract().getName()+")", in1, in2, DriverStrategy.MERGE, this.keys1, this.keys2, inputOrders);
  }
View Full Code Here

TOP

Related Classes of eu.stratosphere.compiler.plan.DualInputPlanNode

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.