Examples of ReduceWork

org.apache.hadoop.hive.ql.plan.ReduceWork
ReduceWork represents all the information used to run a reduce task on the cluster. It is first used when the query planner breaks the logical plan into tasks and used throughout physical optimization to track reduce-side operator plans, schema info about key/value pairs, etc ExecDriver will serialize the contents of this class and make sure it is distributed on the cluster. The ExecReducer will ultimately deserialize this class on the data nodes and setup it's operator pipeline accordingly. This class is also used in the explain command any property with the appropriate annotation will be displayed in the explain output.

Examples of org.apache.hadoop.hive.ql.plan.ReduceWork

      }
    }
  }


  private void checkMRReducer(String taskName, MapredWork mrWrk) throws SemanticException {
    ReduceWork rWrk = mrWrk.getReduceWork();
    if ( rWrk == null) {
      return;
    }
    Operator<? extends OperatorDesc> reducer = rWrk.getReducer();
    if ( reducer instanceof JoinOperator ) {
      BaseWork prntWork = mrWrk.getMapWork();
      checkForCrossProduct(taskName, reducer,
          new ExtractReduceSinkInfo(null).analyze(prntWork));
    }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.ReduceWork

    return true;
  }


  private JoinOperator getJoinOp(MapRedTask task) throws SemanticException {
    MapWork mWork = task.getWork().getMapWork();
    ReduceWork rWork = task.getWork().getReduceWork();
    if (rWork == null) {
      return null;
    }
    Operator<? extends OperatorDesc> reducerOp = rWork.getReducer();
    if (reducerOp instanceof JoinOperator) {
      /* Is any operator present, which prevents the conversion */
      Map<String, Operator<? extends OperatorDesc>> aliasToWork = mWork.getAliasToWork();
      for (Operator<? extends OperatorDesc> op : aliasToWork.values()) {
        if (!checkOperatorOKMapJoinConversion(op)) {

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.ReduceWork

        getMapJoinOperator(newTask, newWork, smbJoinOp, joinTree, bigTablePosition);


    // The reducer needs to be restored - Consider a query like:
    // select count(*) FROM bucket_big a JOIN bucket_small b ON a.key = b.key;
    // The reducer contains a groupby, which needs to be restored.
    ReduceWork rWork = newWork.getReduceWork();


    // create the local work for this plan
    MapJoinProcessor.genLocalWorkForMapJoin(newWork, newMapJoinOp, bigTablePosition);


    // restore the reducer

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.ReduceWork


      // need to add this branch to the key + value info
      assert operator instanceof ReduceSinkOperator
        && followingWork instanceof ReduceWork;
      ReduceSinkOperator rs = (ReduceSinkOperator) operator;
      ReduceWork rWork = (ReduceWork) followingWork;
      GenMapRedUtils.setKeyAndValueDesc(rWork, rs);


      // remember which parent belongs to which tag
      rWork.getTagToInput().put(rs.getConf().getTag(), work.getName());


      // remember the output name of the reduce sink
      rs.getConf().setOutputName(rWork.getName());


      if (!context.connectedReduceSinks.contains(rs)) {
        // add dependency between the two work items
        TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE);
        tezWork.connect(work, rWork, edgeProp);

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.ReduceWork

    HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap =
        opProcCtx.getOpTaskMap();
    Operator<? extends OperatorDesc> currTopOp = opProcCtx.getCurrTopOp();


    opTaskMap.put(reducer, currTask);
    plan.setReduceWork(new ReduceWork());
    plan.getReduceWork().setReducer(reducer);
    ReduceSinkDesc desc = op.getConf();


    plan.getReduceWork().setNumReduceTasks(desc.getNumReducers());

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.ReduceWork

    HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap =
        opProcCtx.getOpTaskMap();


    opTaskMap.put(reducer, unionTask);


    plan.setReduceWork(new ReduceWork());
    plan.getReduceWork().setReducer(reducer);
    plan.getReduceWork().setReducer(reducer);
    ReduceSinkDesc desc = op.getConf();


    plan.getReduceWork().setNumReduceTasks(desc.getNumReducers());

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.ReduceWork

    Task<? extends Serializable> childTask = TaskFactory.get(childPlan, parseCtx
        .getConf());
    Operator<? extends OperatorDesc> reducer = cRS.getChildOperators().get(0);


    // Add the reducer
    ReduceWork rWork = new ReduceWork();
    childPlan.setReduceWork(rWork);
    rWork.setReducer(reducer);
    ReduceSinkDesc desc = cRS.getConf();
    childPlan.getReduceWork().setNumReduceTasks(new Integer(desc.getNumReducers()));


    opProcCtx.getOpTaskMap().put(reducer, childTask);

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.ReduceWork

        } else {
          // There are separate configuration parameters to control whether to
          // merge for a map-only job
          // or for a map-reduce job
          if (currTask.getWork() instanceof MapredWork) {  
            ReduceWork reduceWork = ((MapredWork) currTask.getWork()).getReduceWork();
            boolean mergeMapOnly =
              hconf.getBoolVar(ConfVars.HIVEMERGEMAPFILES) && reduceWork == null;
            boolean mergeMapRed =
              hconf.getBoolVar(ConfVars.HIVEMERGEMAPREDFILES) &&
              reduceWork != null;

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.ReduceWork


    BaseWork w = work.getAllWork().get(1);
    assertTrue(w instanceof ReduceWork);
    assertTrue(work.getParents(w).contains(work.getAllWork().get(0)));


    ReduceWork rw = (ReduceWork)w;


    // need to make sure names are set for tez to connect things right
    assertNotNull(w.getName());


    // map work should start with our ts op
    assertSame(rw.getReducer(),fs);


    // should have severed the ties
    assertEquals(fs.getParentOperators().size(),0);
  }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.ReduceWork

        .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")),
        Utilities.makeList(getStringColumn("value")), outputColumns, true,
        -1, 1, -1));


    addMapWork(mr, src, "a", op1);
    ReduceWork rWork = new ReduceWork();
    rWork.setNumReduceTasks(Integer.valueOf(1));
    rWork.setKeyDesc(op1.getConf().getKeySerializeInfo());
    rWork.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo());
    mr.setReduceWork(rWork);


    // reduce side work
    Operator<FileSinkDesc> op3 = OperatorFactory.get(new FileSinkDesc(new Path(tmpdir + File.separator
        + "mapredplan1.out"), Utilities.defaultTd, false));


    Operator<ExtractDesc> op2 = OperatorFactory.get(new ExtractDesc(
        getStringColumn(Utilities.ReduceField.VALUE.toString())), op3);


    rWork.setReducer(op2);
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.