Examples of org.apache.hadoop.hive.ql.exec.mr.ExecDriver

Package org.apache.hadoop.hive.ql.exec.mr

Examples of org.apache.hadoop.hive.ql.exec.mr.ExecDriver

org.apache.hadoop.hive.ql.exec.mr.ExecDriver
ExecDriver is the central class in co-ordinating execution of any map-reduce task. It's main responsabilities are: - Converting the plan (MapredWork) into a MR Job (JobConf) - Submitting a MR job to the cluster via JobClient and ExecHelper - Executing MR job in local execution mode (where applicable)

      stage.setStageType(task.getType());
      query.addToStageList(stage);


      if (task instanceof ExecDriver) {
        // populate map task
        ExecDriver mrTask = (ExecDriver) task;
        org.apache.hadoop.hive.ql.plan.api.Task mapTask =
          new org.apache.hadoop.hive.ql.plan.api.Task();
        mapTask.setTaskId(stage.getStageId() + "_MAP");
        mapTask.setTaskType(TaskType.MAP);
        stage.addToTaskList(mapTask);
        populateOperatorGraph(mapTask, mrTask.getWork().getMapWork().getAliasToWork()
            .values());


        // populate reduce task
        if (mrTask.hasReduce()) {
          org.apache.hadoop.hive.ql.plan.api.Task reduceTask =
            new org.apache.hadoop.hive.ql.plan.api.Task();
          reduceTask.setTaskId(stage.getStageId() + "_REDUCE");
          reduceTask.setTaskType(TaskType.REDUCE);
          stage.addToTaskList(reduceTask);
          Collection<Operator<? extends OperatorDesc>> reducerTopOps =
            new ArrayList<Operator<? extends OperatorDesc>>();
          reducerTopOps.add(mrTask.getWork().getReduceWork().getReducer());
          populateOperatorGraph(reduceTask, reducerTopOps);
        }
      } else {
        org.apache.hadoop.hive.ql.plan.api.Task otherTask =
          new org.apache.hadoop.hive.ql.plan.api.Task();

View Full Code Here

      }
      if (task.done()) {
        done.add(task.getId());
      }
      if (task instanceof ExecDriver) {
        ExecDriver mrTask = (ExecDriver) task;
        extractOperatorCounters(mrTask.getWork().getMapWork().getAliasToWork().values(),
            task.getId() + "_MAP");
        if (mrTask.mapStarted()) {
          started.add(task.getId() + "_MAP");
        }
        if (mrTask.mapDone()) {
          done.add(task.getId() + "_MAP");
        }
        if (mrTask.hasReduce()) {
          Collection<Operator<? extends OperatorDesc>> reducerTopOps =
            new ArrayList<Operator<? extends OperatorDesc>>();
          reducerTopOps.add(mrTask.getWork().getReduceWork().getReducer());
          extractOperatorCounters(reducerTopOps, task.getId() + "_REDUCE");
          if (mrTask.reduceStarted()) {
            started.add(task.getId() + "_REDUCE");
          }
          if (mrTask.reduceDone()) {
            done.add(task.getId() + "_REDUCE");
          }
        }
      } else if (task instanceof ConditionalTask) {
        ConditionalTask cTask = (ConditionalTask) task;

View Full Code Here

      stage.setStageType(task.getType());
      query.addToStageList(stage);


      if (task instanceof ExecDriver) {
        // populate map task
        ExecDriver mrTask = (ExecDriver) task;
        org.apache.hadoop.hive.ql.plan.api.Task mapTask =
          new org.apache.hadoop.hive.ql.plan.api.Task();
        mapTask.setTaskId(stage.getStageId() + "_MAP");
        mapTask.setTaskType(TaskType.MAP);
        stage.addToTaskList(mapTask);
        populateOperatorGraph(mapTask, mrTask.getWork().getMapWork().getAliasToWork()
            .values());


        // populate reduce task
        if (mrTask.hasReduce()) {
          org.apache.hadoop.hive.ql.plan.api.Task reduceTask =
            new org.apache.hadoop.hive.ql.plan.api.Task();
          reduceTask.setTaskId(stage.getStageId() + "_REDUCE");
          reduceTask.setTaskType(TaskType.REDUCE);
          stage.addToTaskList(reduceTask);
          Collection<Operator<? extends OperatorDesc>> reducerTopOps =
            new ArrayList<Operator<? extends OperatorDesc>>();
          reducerTopOps.add(mrTask.getWork().getReduceWork().getReducer());
          populateOperatorGraph(reduceTask, reducerTopOps);
        }
      } else {
        org.apache.hadoop.hive.ql.plan.api.Task otherTask =
          new org.apache.hadoop.hive.ql.plan.api.Task();

View Full Code Here

      }
      if (task.done()) {
        done.add(task.getId());
      }
      if (task instanceof ExecDriver) {
        ExecDriver mrTask = (ExecDriver) task;
        if (mrTask.mapStarted()) {
          started.add(task.getId() + "_MAP");
        }
        if (mrTask.mapDone()) {
          done.add(task.getId() + "_MAP");
        }
        if (mrTask.hasReduce()) {
          if (mrTask.reduceStarted()) {
            started.add(task.getId() + "_REDUCE");
          }
          if (mrTask.reduceDone()) {
            done.add(task.getId() + "_REDUCE");
          }
        }
      } else if (task instanceof ConditionalTask) {
        ConditionalTask cTask = (ConditionalTask) task;

View Full Code Here

      stage.setStageType(task.getType());
      query.addToStageList(stage);


      if (task instanceof ExecDriver) {
        // populate map task
        ExecDriver mrTask = (ExecDriver) task;
        org.apache.hadoop.hive.ql.plan.api.Task mapTask =
          new org.apache.hadoop.hive.ql.plan.api.Task();
        mapTask.setTaskId(stage.getStageId() + "_MAP");
        mapTask.setTaskType(TaskType.MAP);
        stage.addToTaskList(mapTask);
        populateOperatorGraph(mapTask, mrTask.getWork().getMapWork().getAliasToWork()
            .values());


        // populate reduce task
        if (mrTask.hasReduce()) {
          org.apache.hadoop.hive.ql.plan.api.Task reduceTask =
            new org.apache.hadoop.hive.ql.plan.api.Task();
          reduceTask.setTaskId(stage.getStageId() + "_REDUCE");
          reduceTask.setTaskType(TaskType.REDUCE);
          stage.addToTaskList(reduceTask);
          Collection<Operator<? extends OperatorDesc>> reducerTopOps =
            new ArrayList<Operator<? extends OperatorDesc>>();
          reducerTopOps.add(mrTask.getWork().getReduceWork().getReducer());
          populateOperatorGraph(reduceTask, reducerTopOps);
        }
      } else {
        org.apache.hadoop.hive.ql.plan.api.Task otherTask =
          new org.apache.hadoop.hive.ql.plan.api.Task();

View Full Code Here

      }
      if (task.done()) {
        done.add(task.getId());
      }
      if (task instanceof ExecDriver) {
        ExecDriver mrTask = (ExecDriver) task;
        if (mrTask.mapStarted()) {
          started.add(task.getId() + "_MAP");
        }
        if (mrTask.mapDone()) {
          done.add(task.getId() + "_MAP");
        }
        if (mrTask.hasReduce()) {
          if (mrTask.reduceStarted()) {
            started.add(task.getId() + "_REDUCE");
          }
          if (mrTask.reduceDone()) {
            done.add(task.getId() + "_REDUCE");
          }
        }
      } else if (task instanceof ConditionalTask) {
        ConditionalTask cTask = (ConditionalTask) task;

View Full Code Here

      stage.setStageType(task.getType());
      query.addToStageList(stage);


      if (task instanceof ExecDriver) {
        // populate map task
        ExecDriver mrTask = (ExecDriver) task;
        org.apache.hadoop.hive.ql.plan.api.Task mapTask =
          new org.apache.hadoop.hive.ql.plan.api.Task();
        mapTask.setTaskId(stage.getStageId() + "_MAP");
        mapTask.setTaskType(TaskType.MAP);
        stage.addToTaskList(mapTask);
        populateOperatorGraph(mapTask, mrTask.getWork().getMapWork().getAliasToWork()
            .values());


        // populate reduce task
        if (mrTask.hasReduce()) {
          org.apache.hadoop.hive.ql.plan.api.Task reduceTask =
            new org.apache.hadoop.hive.ql.plan.api.Task();
          reduceTask.setTaskId(stage.getStageId() + "_REDUCE");
          reduceTask.setTaskType(TaskType.REDUCE);
          stage.addToTaskList(reduceTask);
          Collection<Operator<? extends OperatorDesc>> reducerTopOps =
            new ArrayList<Operator<? extends OperatorDesc>>();
          reducerTopOps.add(mrTask.getWork().getReduceWork().getReducer());
          populateOperatorGraph(reduceTask, reducerTopOps);
        }
      } else {
        org.apache.hadoop.hive.ql.plan.api.Task otherTask =
          new org.apache.hadoop.hive.ql.plan.api.Task();

View Full Code Here

      }
      if (task.done()) {
        done.add(task.getId());
      }
      if (task instanceof ExecDriver) {
        ExecDriver mrTask = (ExecDriver) task;
        if (mrTask.mapStarted()) {
          started.add(task.getId() + "_MAP");
        }
        if (mrTask.mapDone()) {
          done.add(task.getId() + "_MAP");
        }
        if (mrTask.hasReduce()) {
          if (mrTask.reduceStarted()) {
            started.add(task.getId() + "_REDUCE");
          }
          if (mrTask.reduceDone()) {
            done.add(task.getId() + "_REDUCE");
          }
        }
      } else if (task instanceof ConditionalTask) {
        ConditionalTask cTask = (ConditionalTask) task;

View Full Code Here

      stage.setStageType(task.getType());
      query.addToStageList(stage);


      if (task instanceof ExecDriver) {
        // populate map task
        ExecDriver mrTask = (ExecDriver) task;
        org.apache.hadoop.hive.ql.plan.api.Task mapTask =
          new org.apache.hadoop.hive.ql.plan.api.Task();
        mapTask.setTaskId(stage.getStageId() + "_MAP");
        mapTask.setTaskType(TaskType.MAP);
        stage.addToTaskList(mapTask);
        populateOperatorGraph(mapTask, mrTask.getWork().getMapWork().getAliasToWork()
            .values());


        // populate reduce task
        if (mrTask.hasReduce()) {
          org.apache.hadoop.hive.ql.plan.api.Task reduceTask =
            new org.apache.hadoop.hive.ql.plan.api.Task();
          reduceTask.setTaskId(stage.getStageId() + "_REDUCE");
          reduceTask.setTaskType(TaskType.REDUCE);
          stage.addToTaskList(reduceTask);
          Collection<Operator<? extends OperatorDesc>> reducerTopOps =
            new ArrayList<Operator<? extends OperatorDesc>>();
          reducerTopOps.add(mrTask.getWork().getReduceWork().getReducer());
          populateOperatorGraph(reduceTask, reducerTopOps);
        }
      } else {
        org.apache.hadoop.hive.ql.plan.api.Task otherTask =
          new org.apache.hadoop.hive.ql.plan.api.Task();

View Full Code Here

      }
      if (task.done()) {
        done.add(task.getId());
      }
      if (task instanceof ExecDriver) {
        ExecDriver mrTask = (ExecDriver) task;
        extractOperatorCounters(mrTask.getWork().getMapWork().getAliasToWork().values(),
            task.getId() + "_MAP");
        if (mrTask.mapStarted()) {
          started.add(task.getId() + "_MAP");
        }
        if (mrTask.mapDone()) {
          done.add(task.getId() + "_MAP");
        }
        if (mrTask.hasReduce()) {
          Collection<Operator<? extends OperatorDesc>> reducerTopOps =
            new ArrayList<Operator<? extends OperatorDesc>>();
          reducerTopOps.add(mrTask.getWork().getReduceWork().getReducer());
          extractOperatorCounters(reducerTopOps, task.getId() + "_REDUCE");
          if (mrTask.reduceStarted()) {
            started.add(task.getId() + "_REDUCE");
          }
          if (mrTask.reduceDone()) {
            done.add(task.getId() + "_REDUCE");
          }
        }
      } else if (task instanceof ConditionalTask) {
        ConditionalTask cTask = (ConditionalTask) task;

View Full Code Here

TOP

Related Classes of org.apache.hadoop.hive.ql.exec.mr.ExecDriver

com.facebook.presto.hive.shaded.org.apache.commons.logging.Log

org.apache.commons.logging.Log

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.Path

org.apache.hadoop.hive.conf.HiveConf

org.apache.hadoop.hive.ql.Context

org.apache.hadoop.hive.ql.DriverContext

org.apache.hadoop.hive.ql.exec.FetchOperator

org.apache.hadoop.hive.ql.exec.JobCloseFeedBack

org.apache.hadoop.hive.ql.exec.PartitionKeySampler

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.