Examples of UnionOperator

Examples of operators.UnionOperator

27
28
29
30
31
32
33
34
35
36
37
    @Override
    public Result execute() {
        Result inResult1 = getInputTerm1().execute();
        Result inResult2 = getInputTerm2().execute();
        UnionOperator oper = new UnionOperator(Utils.getCaf(), inResult1.getResultRelation(), inResult2.getResultRelation());
        inResult1.extend(inResult2);
        inResult1.extend(oper);
        storeResult(inResult1);
        return inResult1;
    }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.UnionOperator

318
319
320
321
322
323
324
325
326
327
328
   * for the union. The plan has already been created.
   */
  public static void initUnionPlan(GenMRProcContext opProcCtx,
      Task<? extends Serializable> currTask, boolean local) {
    MapredWork plan = (MapredWork) currTask.getWork();
    UnionOperator currUnionOp = opProcCtx.getCurrUnionOp();
    assert currUnionOp != null;
    GenMRUnionCtx uCtx = opProcCtx.getUnionTask(currUnionOp);
    assert uCtx != null;

    List<String> taskTmpDirLst = uCtx.getTaskTmpDir();
View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.UnionOperator

581
582
583
584
585
586
587
588
589
590
591
      return dest;

    }

    UnionOperator currUnionOp = ctx.getCurrUnionOp();

    if (currUnionOp != null) {
      opTaskMap.put(null, currTask);
      GenMapRedUtils.initUnionPlan(ctx, currTask, false);
      return dest;
View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.UnionOperator

50
51
52
53
54
55
56
57
58
59
60
    boolean ret = false;

    Task<? extends Serializable> mvTask = ctx.getMvTask();
    Task<? extends Serializable> currTask = ctx.getCurrTask();
    Operator<? extends Serializable> currTopOp = ctx.getCurrTopOp();
    UnionOperator currUnionOp = ctx.getCurrUnionOp();
    String currAliasId = ctx.getCurrAliasId();
    HashMap<Operator<? extends Serializable>, Task<? extends Serializable>> opTaskMap = ctx.getOpTaskMap();
    List<Operator<? extends Serializable>> seenOps = ctx.getSeenOps();
    List<Task<? extends Serializable>>  rootTasks = ctx.getRootTasks();
View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.UnionOperator

140
141
142
143
144
145
146
147
148
149
150
   * It is a idempotent function to add various intermediate files as the source for the
   * union. The plan has already been created.
   */
  public static void initUnionPlan(GenMRProcContext opProcCtx, Task<? extends Serializable> currTask) {
    mapredWork plan = (mapredWork) currTask.getWork();
    UnionOperator currUnionOp = opProcCtx.getCurrUnionOp();
    assert currUnionOp != null;
    GenMRUnionCtx uCtx = opProcCtx.getUnionTask(currUnionOp);
    assert uCtx != null;

    List<String>    taskTmpDirLst = uCtx.getTaskTmpDir();
View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.UnionOperator

51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
  public static class MapRedUnion implements NodeProcessor {

    @Override
    public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
        Object... nodeOutputs) throws SemanticException {
      UnionOperator union = (UnionOperator)nd;
      UnionProcContext ctx = (UnionProcContext) procCtx;

      // find the branch on which this processor was invoked
      int pos = getPositionParent(union, stack);
      UnionParseContext uCtx = ctx.getUnionParseContext(union);
      if (uCtx == null)
        uCtx = new UnionParseContext(union.getConf().getNumInputs());

      ctx.setMapOnlySubq(false);
      uCtx.setMapOnlySubq(pos, false);
      uCtx.setRootTask(pos, false);
      ctx.setUnionParseContext(union, uCtx);
View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.UnionOperator

76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
  public static class MapUnion implements NodeProcessor {

    @Override
    public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
        Object... nodeOutputs) throws SemanticException {
      UnionOperator union = (UnionOperator)nd;
      UnionProcContext ctx = (UnionProcContext) procCtx;

      // find the branch on which this processor was invoked
      int pos = getPositionParent(union, stack);
      UnionParseContext uCtx = ctx.getUnionParseContext(union);
      if (uCtx == null)
        uCtx = new UnionParseContext(union.getConf().getNumInputs());

      uCtx.setMapOnlySubq(pos, true);
      uCtx.setRootTask(pos, true);
      ctx.setUnionParseContext(union, uCtx);
      return null;
View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.UnionOperator

100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
  public static class UnknownUnion implements NodeProcessor {

    @Override
    public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
        Object... nodeOutputs) throws SemanticException {
      UnionOperator union = (UnionOperator)nd;
      UnionProcContext ctx = (UnionProcContext) procCtx;

      // find the branch on which this processor was invoked
      int pos = getPositionParent(union, stack);
      UnionParseContext uCtx = ctx.getUnionParseContext(union);
      if (uCtx == null)
        uCtx = new UnionParseContext(union.getConf().getNumInputs());

      uCtx.setMapOnlySubq(pos, true);
      uCtx.setRootTask(pos, false);
      ctx.setUnionParseContext(union, uCtx);
      return null;
View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.UnionOperator

65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
   * This can be optimized later on.
   * @param nd the file sink operator encountered
   * @param opProcCtx context
   */
  public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx opProcCtx, Object... nodeOutputs) throws SemanticException {
    UnionOperator union = (UnionOperator)nd;
    GenMRProcContext ctx = (GenMRProcContext)opProcCtx;
    ParseContext parseCtx = ctx.getParseCtx();
    UnionProcContext uCtx = parseCtx.getUCtx();

    // Map-only subqueries can be optimized in future to not write to a file in future
    Map<Operator<? extends Serializable>, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx();

    // The plan needs to be broken only if one of the sub-queries involve a map-reduce job
    if (uCtx.isMapOnlySubq()) {
      mapCurrCtx.put((Operator<? extends Serializable>)nd, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(), ctx.getCurrAliasId()));
      return null;
    }

    ctx.setCurrUnionOp(union);

    UnionParseContext uPrsCtx = uCtx.getUnionParseContext(union);
    assert uPrsCtx != null;

    Task<? extends Serializable> currTask = ctx.getCurrTask();
    int pos = UnionProcFactory.getPositionParent(union, stack);

    // is the current task a root task
    if (uPrsCtx.getRootTask(pos) && (!ctx.getRootTasks().contains(currTask)))
      ctx.getRootTasks().add(currTask);
   
    GenMRUnionCtx uCtxTask = ctx.getUnionTask(union);
    Task<? extends Serializable> uTask = null;

    pos = UnionProcFactory.getPositionParent(union, stack);
    Operator<? extends Serializable> parent = union.getParentOperators().get(pos);  
    mapredWork uPlan = null;

    // union is encountered for the first time
    if (uCtxTask == null) {
      uCtxTask = new GenMRUnionCtx();
View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.UnionOperator

367
368
369
370
371
372
373
374
375
376
377
378
          } else {
            return correlatedReduceSinkOperators;
          }
        }
        correlatedReduceSinkOperators.addAll(corrRSs);
        UnionOperator union = (UnionOperator)current;
        union.getConf().setAllInputsInSameReducer(true);
      } else if (current.getColumnExprMap() == null && !(current instanceof ReduceSinkOperator)) {
        for (Operator<? extends OperatorDesc> parent : current.getParentOperators()) {
          correlatedReduceSinkOperators.addAll(
              findCorrelatedReduceSinkOperators(
                  current, childKeyCols, childPartitionCols, childRSOrder, parent, correlation));
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.