Package org.apache.hadoop.mapred.TaskTracker

Examples of org.apache.hadoop.mapred.TaskTracker.TaskInProgress


    for (int i = 0; i < MAP_SLOTS*5; i++) {
      JobConf taskConf = new JobConf(ttConf);
      TaskAttemptID attemptID = new TaskAttemptID("test", 0, TaskType.MAP, i, 0);
      Task task = new MapTask(null, attemptID, i, null, 1);
      task.setConf(taskConf);
      TaskInProgress tip = tt.new TaskInProgress(task, taskConf);
      File pidFile = new File(TEST_DIR, "pid_" + i);
      final TaskRunner taskRunner = task.createRunner(tt, tip);
      // launch a jvm which sleeps for 60 seconds
      final Vector<String> vargs = new Vector<String>(2);
      vargs.add(writeScript("script_" + i, "echo hi\n", pidFile).getAbsolutePath());
View Full Code Here


    String jtId = "test";
    TaskAttemptID attemptID = new TaskAttemptID(jtId, 1, TaskType.MAP, 0, 0);
    Task task = new MapTask(null, attemptID, 0, null, 5);
    mapLauncher.addToTaskQueue(new LaunchTaskAction(task));
    // verify that task is added to runningTasks
    TaskInProgress killTip = tt.runningTasks.get(attemptID);
    assertNotNull(killTip);

    // wait for a while for launcher to pick up the task
    // this loop waits atmost for 30 seconds
    for (int i = 0; i < 300; i++) {
      if (mapLauncher.getNumWaitingTasksToLaunch() == 0) {
        break;
      }
      UtilsForTests.waitFor(100);
    }
    assertEquals("Launcher didnt pick up the task " + attemptID + "to launch",
        0, mapLauncher.getNumWaitingTasksToLaunch());

    // Now, that launcher has picked up the task, it waits until all five slots
    // are available. i.e. it waits for-ever
    // lets kill the task so that map launcher comes out
    tt.processKillTaskAction(new KillTaskAction(attemptID));
    assertEquals(TaskStatus.State.KILLED, killTip.getRunState());

    // launch another attempt which requires only one slot
    TaskAttemptID runningAttemptID = new TaskAttemptID(jtId, 1, TaskType.MAP,
        0, expectedLaunchAttemptId);
    mapLauncher.addToTaskQueue(new LaunchTaskAction(new MapTask(null,
        runningAttemptID, 0, null, 1)));
    TaskInProgress runningTip = tt.runningTasks.get(runningAttemptID);
    assertNotNull(runningTip);

    // wait for a while for the task to be launched
    // this loop waits at most for 30 seconds
    for (int i = 0; i < 300; i++) {
      if (runningTip.getRunState().equals(TaskStatus.State.RUNNING)) {
        break;
      }
      UtilsForTests.waitFor(100);
    }

    // verify that the task went to running
    assertEquals(TaskStatus.State.RUNNING, runningTip.getRunState());
  }
View Full Code Here

    List<TaskAttemptID> tasksToExclude = new ArrayList<TaskAttemptID>();
    // Find tasks to kill so as to get memory usage under limits.
    while (memoryStillInUsage > maxMemoryAllowedForAllTasks) {
      // Exclude tasks that are already marked for
      // killing.
      TaskInProgress task = taskTracker.findTaskToKill(tasksToExclude);
      if (task == null) {
        break; // couldn't find any more tasks to kill.
      }

      TaskAttemptID tid = task.getTask().getTaskID();
      if (processTreeInfoMap.containsKey(tid)) {
        ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid);
        ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
        memoryStillInUsage -= pTree.getCumulativeVmem();
        tasksToKill.add(tid);
View Full Code Here

   * Check if a task can be killed to increase free memory
   * @param tid task attempt ID
   * @return true if the task can be killed
   */
  private boolean isKillable(TaskAttemptID tid) {
      TaskInProgress tip = taskTracker.runningTasks.get(tid);
      return tip != null && !tip.wasKilled() &&
             (tip.getRunState() == TaskStatus.State.RUNNING ||
              tip.getRunState() == TaskStatus.State.COMMIT_PENDING);
  }
View Full Code Here

        tipQueue.put(ttf);
        continue;
      }

      // Finish the task
      TaskInProgress tip = ttf.getTip();
      ttf.finishTip();

      // Also clean up the mapper wait thread map for reducers. It should exist
      // for reduce tasks that are not cleanup tasks
      if (!tip.getTask().isMapTask() &&
          !tip.getTask().isTaskCleanupTask() &&
          !tip.getTask().isJobCleanupTask() &&
          !tip.getTask().isJobSetupTask()) {
        if (!mapperWaitThreadMap.containsKey(tip)) {
          throw new RuntimeException("Unable to find mapper wait thread for " +
              tip.getTask().getTaskID() + " job " + tip.getTask().getJobID());
        }
        LOG.debug("Removing mapper wait thread for " +
            tip.getTask().getTaskID() + " job " + tip.getTask().getJobID());
        mapperWaitThreadMap.remove(tip);
      } else if (mapperWaitThreadMap.containsKey(tip)) {
        throw new RuntimeException("Mapper wait thread exists for" +
            tip.getTask().getTaskID() + " job " + tip.getTask().getJobID() +
            " when it shouldn't!");
      }
    }
  }
View Full Code Here

   * Check if a task can be killed to increase free memory
   * @param tid task attempt ID
   * @return true if the task can be killed
   */
  private boolean isKillable(TaskAttemptID tid) {
    TaskInProgress tip = taskTracker.runningTasks.get(tid);
    return tip != null && !tip.wasKilled() &&
      (tip.getRunState() == TaskStatus.State.RUNNING ||
      tip.getRunState() == TaskStatus.State.COMMIT_PENDING);
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapred.TaskTracker.TaskInProgress

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.