Examples of TaskInProgress


Examples of org.apache.hadoop.mapred.TaskTracker.TaskInProgress

    // Find tasks to kill so as to get memory usage under limits.
    while (memoryStillInUsage > maxMemoryAllowedForAllTasks) {
      // Exclude tasks that are already marked for killing.
      // Note that we do not need to call isKillable() here because the logic
      // is contained in taskTracker.findTaskToKill()
      TaskInProgress task = taskTracker.findTaskToKill(tasksToExclude);
      if (task == null) {
        break; // couldn't find any more tasks to kill.
      }

      TaskAttemptID tid = task.getTask().getTaskID();
      if (processTreeInfoMap.containsKey(tid)) {
        ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid);
        ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
        memoryStillInUsage -= pTree.getCumulativeVmem();
        tasksToKill.add(tid);
View Full Code Here

Examples of org.apache.hadoop.mapred.TaskTracker.TaskInProgress

   * Kill the task and clean up ProcessTreeInfo
   * @param tid task attempt ID of the task to be killed.
   * @param msg diagnostics message
   */
  private void killTask(TaskAttemptID tid, String msg) {
    TaskInProgress tip = taskTracker.getRunningTask(tid);
    if (tip != null) {
      //for the task identified to be killed update taskDiagnostic
      TaskStatus taskStatus = tip.getStatus();
      taskStatus.setDiagnosticInfo(msg);
    }
    // Kill the task and mark it as killed.
    taskTracker.cleanUpOverMemoryTask(tid, false, msg);
    // Now destroy the ProcessTree, remove it from monitoring map.
View Full Code Here

Examples of org.apache.hadoop.mapred.TaskTracker.TaskInProgress

   * Check if a task can be killed to increase free memory
   * @param tid task attempt ID
   * @return true if the task can be killed
   */
  private boolean isKillable(TaskAttemptID tid) {
      TaskInProgress tip = taskTracker.getRunningTask(tid);
      return tip != null && !tip.wasKilled() &&
             (tip.getRunState() == TaskStatus.State.RUNNING ||
              tip.getRunState() == TaskStatus.State.COMMIT_PENDING);
  }
View Full Code Here

Examples of org.apache.hadoop.mapred.TaskTracker.TaskInProgress

        new TaskAttemptID(jtIdentifier, jobId.getId(), TaskType.MAP, 1, 0);
    createTask();

    // mimic register task
    // create the tip
    tip = tracker.new TaskInProgress(task, trackerFConf);
  }
View Full Code Here

Examples of org.apache.hadoop.mapred.TaskTracker.TaskInProgress

   
    // create a task cleanup attempt
    createTask();
    task.setTaskCleanupTask();
    // register task
    tip = tracker.new TaskInProgress(task, trackerFConf);

    // localize the job again.
    rjob = tracker.localizeJob(tip);
    localizedJobConf = rjob.getJobConf();
    checkJobLocalization();
View Full Code Here

Examples of org.apache.hadoop.mapred.TaskTracker.TaskInProgress

   
    // create a task cleanup attempt
    createTask();
    task.setTaskCleanupTask();
    // register task
    tip = tracker.new TaskInProgress(task, trackerFConf);

    // localize the job again.
    rjob = tracker.localizeJob(tip);
    localizedJobConf = rjob.getJobConf();
    checkJobLocalization();
View Full Code Here

Examples of org.apache.hadoop.mapred.TaskTracker.TaskInProgress

      return;
    }

    task.setTaskCleanupTask();
    // register task
    tip = tracker.new TaskInProgress(task, trackerFConf);

    // localize the job.
    RunningJob rjob = tracker.localizeJob(tip);
    localizedJobConf = rjob.getJobConf();
    checkJobLocalization();
View Full Code Here

Examples of org.apache.hadoop.mapred.TaskTracker.TaskInProgress

    JobConf taskConf = new JobConf(ttConf);
    TaskAttemptID attemptID = new TaskAttemptID("test", 0, true, 0, 0);
    Task task = new MapTask(null, attemptID, 0, null, MAP_SLOTS);
    task.setUser(user);
    task.setConf(taskConf);
    TaskInProgress tip = tt.new TaskInProgress(task, taskConf);
    File pidFile = new File(TEST_DIR, "pid");
    RunningJob rjob = new RunningJob(attemptID.getJobID());
    TaskController taskController = new DefaultTaskController();
    taskController.setConf(ttConf);
    rjob.distCacheMgr =
      new TrackerDistributedCacheManager(ttConf, taskController).
      newTaskDistributedCacheManager(attemptID.getJobID(), taskConf);
    final TaskRunner taskRunner = task.createRunner(tt, tip, rjob);
    // launch a jvm which sleeps for 60 seconds
    final Vector<String> vargs = new Vector<String>(2);
    vargs.add(writeScript("SLEEP", "sleep 60\n", pidFile).getAbsolutePath());
    final File workDir = new File(TEST_DIR, "work");
    final File stdout = new File(TEST_DIR, "stdout");
    final File stderr = new File(TEST_DIR, "stderr");

    // launch the process and wait in a thread, till it finishes
    Thread launcher = new Thread() {
      public void run() {
        try {
          taskRunner.launchJvmAndWait(null, vargs, stdout, stderr, 100,
              workDir);
        } catch (InterruptedException e) {
          e.printStackTrace();
          return;
        } catch (IOException e) {
          e.printStackTrace();
          setThreadCaughtException();
        }
      }
    };
    launcher.start();
    // wait till the jvm is launched
    // this loop waits for at most 1 second
    for (int i = 0; i < 10; i++) {
      if (pidFile.exists()) {
        break;
      }
      UtilsForTests.waitFor(100);
    }
    // assert that the process is launched
    assertTrue("pidFile is not present", pidFile.exists());
   
    // imitate Child code.
    // set pid in jvmManager
    BufferedReader in = new  BufferedReader(new FileReader(pidFile));
    String pid = in.readLine();
    in.close();
    JVMId jvmid = mapJvmManager.runningTaskToJvm.get(taskRunner);
    jvmManager.setPidToJvm(jvmid, pid);

    // kill JvmRunner
    final JvmRunner jvmRunner = mapJvmManager.jvmIdToRunner.get(jvmid);
    Thread killer = new Thread() {
      public void run() {
        try {
          jvmRunner.kill();
        } catch (IOException e) {
          e.printStackTrace();
          setThreadCaughtException();
        } catch (InterruptedException e) {
          e.printStackTrace();
        }
      }
    };
    killer.start();
   
    //wait for a while so that killer thread is started.
    Thread.sleep(100);

    // kill the jvm externally
    taskRunner.kill();

    assertTrue(jvmRunner.killed);

    // launch another jvm and see it finishes properly
    attemptID = new TaskAttemptID("test", 0, true, 0, 1);
    task = new MapTask(null, attemptID, 0, null, MAP_SLOTS);
    task.setUser(user);
    task.setConf(taskConf);
    tip = tt.new TaskInProgress(task, taskConf);
    TaskRunner taskRunner2 = task.createRunner(tt, tip, rjob);
    // build dummy vargs to call ls
    Vector<String> vargs2 = new Vector<String>(1);
    vargs2.add(writeScript("LS", "ls", pidFile).getAbsolutePath());
    File workDir2 = new File(TEST_DIR, "work2");
View Full Code Here

Examples of org.apache.hadoop.mapred.TaskTracker.TaskInProgress

    String jtId = "test";
    TaskAttemptID attemptID = new TaskAttemptID(jtId, 1, true, 0, 0);
    Task task = new MapTask(null, attemptID, 0, null, 2);
    mapLauncher.addToTaskQueue(new LaunchTaskAction(task));
    // verify that task is added to runningTasks
    TaskInProgress runningTip = tt.runningTasks.get(attemptID);
    assertNotNull(runningTip);

    // wait for a while for the first task to start initializing
    // this loop waits at most for 30 seconds
    for (int i = 0; i < 300; i++) {
      if (firstJobStarted == true) {
        break;
      }
      UtilsForTests.waitFor(100);
    }

    // Now start a second task and make sure it doesn't wait while first one initializes
    String secondjtId = "test2";
    TaskAttemptID secondAttemptID = new TaskAttemptID(secondjtId, 1, true, 0, 0);
    Task secondTask = new MapTask(null, secondAttemptID, 0, null, 2);
    mapLauncher.addToTaskQueue(new LaunchTaskAction(secondTask));
    // verify that task is added to runningTasks
    TaskInProgress secondRunningTip = tt.runningTasks.get(secondAttemptID);
    assertNotNull(secondRunningTip);

    // wait for a while for the second task to be launched
    // this loop waits at most for 30 seconds
    for (int i = 0; i < 300; i++) {
View Full Code Here

Examples of org.apache.hadoop.mapred.TaskTracker.TaskInProgress

      }
      TaskRunner taskRunner = jvmToRunningTask.get(jvmId);
      if (taskRunner == null) {
        return false; //JvmId not known.
      }
      TaskInProgress knownTip = taskRunner.getTaskInProgress();
      if (knownTip == tip) { // Valid to compare the addresses ? (or equals)
        return true;
      } else {
        return false;
      }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.