Package org.apache.hadoop.mapreduce.util

Examples of org.apache.hadoop.mapreduce.util.ProcfsBasedProcessTree


      TaskMemoryManagerThread test = new TaskMemoryManagerThread(1000000L,
                                                                5000L);
      // create process trees
      // tree rooted at 100 is over limit immediately, as it is
      // twice over the mem limit.
      ProcfsBasedProcessTree pTree = new ProcfsBasedProcessTree(
                                          "100", true, 100L,
                                          procfsRootDir.getAbsolutePath());
      pTree.getProcessTree();
      assertTrue("tree rooted at 100 should be over limit " +
                    "after first iteration.",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
     
      // the tree rooted at 200 is initially below limit.
      pTree = new ProcfsBasedProcessTree("200", true, 100L,
                                          procfsRootDir.getAbsolutePath());
      pTree.getProcessTree();
      assertFalse("tree rooted at 200 shouldn't be over limit " +
                    "after one iteration.",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      // second iteration - now the tree has been over limit twice,
      // hence it should be declared over limit.
      pTree.getProcessTree();
      assertTrue("tree rooted at 200 should be over limit after 2 iterations",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
     
      // the tree rooted at 600 is never over limit.
      pTree = new ProcfsBasedProcessTree("600", true, 100L,
                                            procfsRootDir.getAbsolutePath());
      pTree.getProcessTree();
      assertFalse("tree rooted at 600 should never be over limit.",
                    test.isProcessTreeOverLimit(pTree, "dummyId", limit));
     
      // another iteration does not make any difference.
      pTree.getProcessTree();
      assertFalse("tree rooted at 600 should never be over limit.",
                    test.isProcessTreeOverLimit(pTree, "dummyId", limit));
    } finally {
      FileUtil.fullyDelete(procfsRootDir);
    }
View Full Code Here


                      .getLong(
                          TTConfig.TT_SLEEP_TIME_BEFORE_SIG_KILL,
                          ProcessTree.DEFAULT_SLEEPTIME_BEFORE_SIGKILL);

              // create process tree object
              ProcfsBasedProcessTree pt =
                  new ProcfsBasedProcessTree(pId,
                      ProcessTree.isSetsidAvailable, sleeptimeBeforeSigkill);
              LOG.debug("Tracking ProcessTree " + pId + " for the first time");

              ptInfo.setPid(pId);
              ptInfo.setProcessTree(pt);
            }
          }
          // End of initializing any uninitialized processTrees

          if (pId == null) {
            continue; // processTree cannot be tracked
          }

          TaskInProgress tip = taskTracker.getRunningTask(tid);
          if (tip == null) {
            continue;
          }
          if (tip.wasKilled()) {
            continue;
          }

          LOG.debug("Constructing ProcessTree for : PID = " + pId + " TID = "
              + tid);
          ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
          pTree = pTree.getProcessTree(); // get the updated process-tree
          ptInfo.setProcessTree(pTree); // update ptInfo with proces-tree of
                                        // updated state
          long currentMemUsage = pTree.getCumulativeVmem();
          long currentRssMemUsage = pTree.getCumulativeRssmem();
          // as processes begin with an age 1, we want to see if there
          // are processes more than 1 iteration old.
          long curMemUsageOfAgedProcesses = pTree.getCumulativeVmem(1);
          long curRssMemUsageOfAgedProcesses = pTree.getCumulativeRssmem(1);
          long limit = ptInfo.getMemLimit();
          long limitPhysical = ptInfo.getMemLimitPhysical();
          LOG.info(String.format(MEMORY_USAGE_STRING,
                                pId, tid.toString(), currentMemUsage, limit,
                                currentRssMemUsage, limitPhysical));

          boolean isMemoryOverLimit = false;
          String msg = "";
          if (doCheckVirtualMemory() &&
              isProcessTreeOverLimit(tid.toString(), currentMemUsage,
                                      curMemUsageOfAgedProcesses, limit)) {
            // Task (the root process) is still alive and overflowing memory.
            // Dump the process-tree and then clean it up.
            msg = "TaskTree [pid=" + pId + ",tipID=" + tid
                    + "] is running beyond memory-limits. Current usage : "
                    + currentMemUsage + "bytes. Limit : " + limit
                    + "bytes. Killing task. \nDump of the process-tree for "
                    + tid + " : \n" + pTree.getProcessTreeDump();
            isMemoryOverLimit = true;
          } else if (doCheckPhysicalMemory() &&
              isProcessTreeOverLimit(tid.toString(), currentRssMemUsage,
                                curRssMemUsageOfAgedProcesses, limitPhysical)) {
            // Task (the root process) is still alive and overflowing memory.
            // Dump the process-tree and then clean it up.
            msg = "TaskTree [pid=" + pId + ",tipID=" + tid
                    + "] is running beyond physical memory-limits."
                    + " Current usage : "
                    + currentRssMemUsage + "bytes. Limit : " + limitPhysical
                    + "bytes. Killing task. \nDump of the process-tree for "
                    + tid + " : \n" + pTree.getProcessTreeDump();
            isMemoryOverLimit = true;
          }

          if (isMemoryOverLimit) {
            // Virtual or physical memory over limit. Fail the task and remove
            // the corresponding process tree
            LOG.warn(msg);
            taskTracker.cleanUpOverMemoryTask(tid, true, msg);
            // Now destroy the ProcessTree, remove it from monitoring map.
            pTree.destroy(true/*in the background*/);
            it.remove();
            LOG.info("Removed ProcessTree with root " + pId);
          } else {
            // Accounting the total memory in usage for all tasks that are still
            // alive and within limits.
View Full Code Here

      }

      TaskAttemptID tid = task.getTask().getTaskID();
      if (processTreeInfoMap.containsKey(tid)) {
        ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid);
        ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
        memoryStillInUsage -= pTree.getCumulativeVmem();
        tasksToKill.add(tid);
      }
      // Exclude this task from next search because it is already
      // considered.
      tasksToExclude.add(tid);
View Full Code Here

   * @param tid the task attempt ID of the task
   * @return rss memory usage in bytes. 0 if the process tree is not available
   */
  private long getTaskCumulativeRssmem(TaskAttemptID tid) {
      ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid);
      ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
      return pTree == null ? 0 : pTree.getCumulativeVmem();
  }
View Full Code Here

    }
    // Kill the task and mark it as killed.
    taskTracker.cleanUpOverMemoryTask(tid, false, msg);
    // Now destroy the ProcessTree, remove it from monitoring map.
    ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid);
    ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
    pTree.destroy(true/*in the background*/);
    processTreeInfoMap.remove(tid);
    LOG.info("Removed ProcessTree with root " + ptInfo.getPID());
  }
View Full Code Here

                      .getLong(
                          TTConfig.TT_SLEEP_TIME_BEFORE_SIG_KILL,
                          ProcessTree.DEFAULT_SLEEPTIME_BEFORE_SIGKILL);

              // create process tree object
              ProcfsBasedProcessTree pt =
                  new ProcfsBasedProcessTree(pId,
                      ProcessTree.isSetsidAvailable, sleeptimeBeforeSigkill);
              LOG.debug("Tracking ProcessTree " + pId + " for the first time");

              ptInfo.setPid(pId);
              ptInfo.setProcessTree(pt);
            }
          }
          // End of initializing any uninitialized processTrees

          if (pId == null) {
            continue; // processTree cannot be tracked
          }

          if (taskTracker.runningTasks.get(tid).wasKilled()) {
            continue; // this task has been killed already
          }

          LOG.debug("Constructing ProcessTree for : PID = " + pId + " TID = "
              + tid);
          ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
          pTree = pTree.getProcessTree(); // get the updated process-tree
          ptInfo.setProcessTree(pTree); // update ptInfo with proces-tree of
                                        // updated state
          long currentMemUsage = pTree.getCumulativeVmem();
          long currentRssMemUsage = pTree.getCumulativeRssmem();
          // as processes begin with an age 1, we want to see if there
          // are processes more than 1 iteration old.
          long curMemUsageOfAgedProcesses = pTree.getCumulativeVmem(1);
          long curRssMemUsageOfAgedProcesses = pTree.getCumulativeRssmem(1);
          long limit = ptInfo.getMemLimit();
          long limitPhysical = ptInfo.getMemLimitPhysical();
          LOG.info(String.format(MEMORY_USAGE_STRING,
                                pId, tid.toString(), currentMemUsage, limit,
                                currentRssMemUsage, limitPhysical));

          boolean isMemoryOverLimit = false;
          String msg = "";
          if (doCheckVirtualMemory() &&
              isProcessTreeOverLimit(tid.toString(), currentMemUsage,
                                      curMemUsageOfAgedProcesses, limit)) {
            // Task (the root process) is still alive and overflowing memory.
            // Dump the process-tree and then clean it up.
            msg = "TaskTree [pid=" + pId + ",tipID=" + tid
                    + "] is running beyond memory-limits. Current usage : "
                    + currentMemUsage + "bytes. Limit : " + limit
                    + "bytes. Killing task. \nDump of the process-tree for "
                    + tid + " : \n" + pTree.getProcessTreeDump();
            isMemoryOverLimit = true;
          } else if (doCheckPhysicalMemory() &&
              isProcessTreeOverLimit(tid.toString(), currentRssMemUsage,
                                curRssMemUsageOfAgedProcesses, limitPhysical)) {
            // Task (the root process) is still alive and overflowing memory.
            // Dump the process-tree and then clean it up.
            msg = "TaskTree [pid=" + pId + ",tipID=" + tid
                    + "] is running beyond physical memory-limits."
                    + " Current usage : "
                    + currentRssMemUsage + "bytes. Limit : " + limitPhysical
                    + "bytes. Killing task. \nDump of the process-tree for "
                    + tid + " : \n" + pTree.getProcessTreeDump();
            isMemoryOverLimit = true;
          }

          if (isMemoryOverLimit) {
            // Virtual or physical memory over limit. Fail the task and remove
            // the corresponding process tree
            LOG.warn(msg);
            taskTracker.cleanUpOverMemoryTask(tid, true, msg);
            // Now destroy the ProcessTree, remove it from monitoring map.
            pTree.destroy(true/*in the background*/);
            it.remove();
            LOG.info("Removed ProcessTree with root " + pId);
          } else {
            // Accounting the total memory in usage for all tasks that are still
            // alive and within limits.
View Full Code Here

      }

      TaskAttemptID tid = task.getTask().getTaskID();
      if (processTreeInfoMap.containsKey(tid)) {
        ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid);
        ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
        memoryStillInUsage -= pTree.getCumulativeVmem();
        tasksToKill.add(tid);
      }
      // Exclude this task from next search because it is already
      // considered.
      tasksToExclude.add(tid);
View Full Code Here

   * @param tid the task attempt ID of the task
   * @return rss memory usage in bytes. 0 if the process tree is not available
   */
  private long getTaskCumulativeRssmem(TaskAttemptID tid) {
      ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid);
      ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
      return pTree == null ? 0 : pTree.getCumulativeVmem();
  }
View Full Code Here

  private void killTask(TaskAttemptID tid, String msg) {
    // Kill the task and mark it as killed.
    taskTracker.cleanUpOverMemoryTask(tid, false, msg);
    // Now destroy the ProcessTree, remove it from monitoring map.
    ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid);
    ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
    pTree.destroy(true/*in the background*/);
    processTreeInfoMap.remove(tid);
    LOG.info("Removed ProcessTree with root " + ptInfo.getPID());
  }
View Full Code Here

      TaskMemoryManagerThread test = new TaskMemoryManagerThread(1000000L,
                                                                5000L);
      // create process trees
      // tree rooted at 100 is over limit immediately, as it is
      // twice over the mem limit.
      ProcfsBasedProcessTree pTree = new ProcfsBasedProcessTree(
                                          "100", true, 100L,
                                          procfsRootDir.getAbsolutePath());
      pTree.getProcessTree();
      assertTrue("tree rooted at 100 should be over limit " +
                    "after first iteration.",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
     
      // the tree rooted at 200 is initially below limit.
      pTree = new ProcfsBasedProcessTree("200", true, 100L,
                                          procfsRootDir.getAbsolutePath());
      pTree.getProcessTree();
      assertFalse("tree rooted at 200 shouldn't be over limit " +
                    "after one iteration.",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      // second iteration - now the tree has been over limit twice,
      // hence it should be declared over limit.
      pTree.getProcessTree();
      assertTrue("tree rooted at 200 should be over limit after 2 iterations",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
     
      // the tree rooted at 600 is never over limit.
      pTree = new ProcfsBasedProcessTree("600", true, 100L,
                                            procfsRootDir.getAbsolutePath());
      pTree.getProcessTree();
      assertFalse("tree rooted at 600 should never be over limit.",
                    test.isProcessTreeOverLimit(pTree, "dummyId", limit));
     
      // another iteration does not make any difference.
      pTree.getProcessTree();
      assertFalse("tree rooted at 600 should never be over limit.",
                    test.isProcessTreeOverLimit(pTree, "dummyId", limit));
    } finally {
      FileUtil.fullyDelete(procfsRootDir);
    }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapreduce.util.ProcfsBasedProcessTree

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.