Package org.apache.hadoop.mapred.ControlGroup

Examples of org.apache.hadoop.mapred.ControlGroup.MemoryControlGroup


      isAvailable = false;
      return;
    }

    String jtRootpath = conf.get(CGROUP_MEM_JT_ROOT, DEFAULT_JT_ROOT);
    jtcgp = new MemoryControlGroup(jtRootpath);
    jtcgp.enableMoveChargeAtImmigrate();

    if (!jtcgp.canControl()) {
      LOG.warn("TaskMemoryControlGroup is disabled because jtgroup doesn't have appropriate permission for "
          + jtRootpath);
      isAvailable = false;
      return;
    }

    String ttRootpath = conf.get(CGROUP_MEM_TT_ROOT, DEFAULT_TT_ROOT);
    ttcgp = new MemoryControlGroup(ttRootpath);
    ttcgp.enableMoveChargeAtImmigrate();

    if (!ttcgp.canControl()) {
      LOG.warn("TaskMemoryControlGroup is disabled because ttgroup doesn't have appropriate permission for "
          + ttRootpath);
      isAvailable = false;
      return;
    }
    if (getPID().equals("")) {
      LOG.warn("TaskMemoryControlGroup is disabled because JVM_PID is not set for TaskTracker");
      isAvailable = false;
      return;
    }
    ttcgp.addToGroup(getPID());

    rootpath = conf.get(CGROUP_MEM_ROOT_PROPERTY, DEFAULT_CGROUP_MEM_ROOT);
    containercgp = new MemoryControlGroup(rootpath);
    if (!containercgp.canControl()) {
      LOG.warn("TaskMemoryControlGroup is disabled because TaskTracker does not have appropriate permission for "
          + rootpath);
      isAvailable = false;
      return;
View Full Code Here


    if (!isAvailable) {
      this.numFailedToAddTask.incrementAndGet();
      return ;
    }

    MemoryControlGroup taskcgp = containercgp.createSubGroup(taskname);
    taskcgp.enableMoveChargeAtImmigrate();
    if (isTaskLimitOn) {
      taskcgp.setMemoryUsageLimit(memoryLimit);
      LOG.info("Task " + taskname + " is added to control group with memory = " +
        memoryLimit );
    } else {
      LOG.info("Task " + taskname + " is added to control group without limit");
    }
    taskcgp.addToGroup(pid);
  }
View Full Code Here

  public void removeTask(String taskname) {
    if (!isAvailable)
      return ;

    MemoryControlGroup taskcgp = containercgp.getSubGroup(taskname);
    trashcgp.addToGroup(taskcgp.getThreadGroupList());
    taskcgp.deleteGroup();
  }
View Full Code Here

  private boolean isAvailable;
  private MemoryControlGroup jtcgp;

  public JobTrackerMemoryControlGroup(Configuration conf) {
    String jtRootpath = conf.get(CGROUP_MEM_JT_ROOT, DEFAULT_JT_ROOT);
    jtcgp = new MemoryControlGroup(jtRootpath);
    jtcgp.enableMoveChargeAtImmigrate();

    if (!jtcgp.canControl()) {
      LOG.warn("MemoryControlGroup is disabled because jtgroup doesn't have appropriate permission for "
          + jtRootpath);
View Full Code Here

        processTreeInfoMap.entrySet().iterator(); it.hasNext();) {
        Map.Entry<TaskAttemptID, CGroupProcessTreeInfo> entry = it.next();
        TaskAttemptID attempt = entry.getKey();
        CGroupProcessTreeInfo pInfo = entry.getValue();
        pInfo.setMemoryUsed(0);
        MemoryControlGroup taskGroup =
          containerGroup.getSubGroup(attempt.toString());
        if (taskGroup == null) {
          LOG.warn(attempt + " has no memory CGroup");
          continue;
        }
       
        long memoryUsed = taskGroup.getRSSMemoryUsage();
        pInfo.setMaxMemoryUsed(memoryUsed);
        memoryUsedByTasks += memoryUsed;
        pInfo.setMemoryUsed(memoryUsed);
       
        if (isTaskLimitOn && isKillable(attempt) &&
            taskGroup.getMemoryUsageLimit() > 0 &&
           (taskGroup.getMemoryUsageLimit() - memoryUsed
            < taskLowMemoryThreshold)) {
          LOG.warn(attempt + " hits the memory threshold " + memoryUsed);
          tasksToKill.add(attempt);
          memoryToRelease -= memoryUsed;
        }
      }
      // kill the tasks which was at the limit
      if (tasksToKill.size() > 0) {
        killTasks();
        taskKilled = true;
      }
      // check container
      long containerMemoryUsed = containerGroup.getRSSMemoryUsage() + memoryUsedByTasks;
      memoryLeft = containerGroup.getMemoryUsageLimit() - containerMemoryUsed;
      if  ((memoryLeft < lowMemoryThreshold) &&
            containerGroup.getMemoryUsageLimit() > 0) {
        LOG.warn("container hits the threshold for RSS memory " + containerMemoryUsed);
        memoryToRelease += lowMemoryThreshold;
      }
      // release memory ASAP
      if (memoryToRelease > 0) {
        LOG.warn("Need to release " + memoryToRelease + " memory");
        failTasksWithMaxMemory(memoryToRelease);
        incOOMNo();
        sleepTime = minMonitoringInterval;
        logAliveTasks();
        continue;
      }
      synchronized (memStat) {
        memStat.setMaxMemoryUsed(containerMemoryUsed);
        memStat.setMemoryUsage(containerMemoryUsed);
        memStat.setMemoryLimit(containerGroup.getMemoryUsageLimit());
        memStat.setLowMemoryThreshold(lowMemoryThreshold);
        memStat.setMemoryUsedByTasks(memoryUsedByTasks);
        long jtMemoryUsed = jtMemoryGroup.getRSSMemoryUsage();
        memStat.setJTMaxMemoryUsed(jtMemoryUsed);
        memStat.setJTMemoryUsage(jtMemoryUsed);
        long ttMemoryUsed = ttMemoryGroup.getRSSMemoryUsage();
        memStat.setTTMaxMemoryUsed(ttMemoryUsed);
        memStat.setTTMemoryUsage(ttMemoryUsed);
      }
      sleepTime = monitoringInterval;
      if (taskKilled) {
        logAliveTasks()
      }
      // remove the very old attempts
      long current = System.currentTimeMillis();
      List<TaskAttemptID> oldTasks = new ArrayList<TaskAttemptID>();
      oldTasks.addAll(processTreeInfoMap.keySet());
      for (TaskAttemptID attempt: oldTasks) {
        CGroupProcessTreeInfo pInfo = processTreeInfoMap.get(attempt);
        if (pInfo == null) {
          continue;
        }
        if (current - pInfo.getCreationTime() >=  oldTaskThreshold) {
          LOG.warn("Remove old entry " + attempt);
          ttMemoryCGroup.removeTask(attempt.toString());
          processTreeInfoMap.remove(attempt);
        } else if (current - pInfo.getCreationTime() >= emptyTaskThreshold) {
          MemoryControlGroup taskGroup =
            containerGroup.getSubGroup(attempt.toString());
          if (taskGroup == null || taskGroup.getMaxMemoryUsage() == 0) {
            LOG.warn("Remove empty entry " + attempt);
            processTreeInfoMap.remove(attempt);
          }
        }
      }
View Full Code Here

        switch (trackerType) {
          case JOB_TRACKER:
            String jtRootpath = conf.get(
                TaskTrackerMemoryControlGroup.CGROUP_MEM_JT_ROOT,
                TaskTrackerMemoryControlGroup.DEFAULT_JT_ROOT);
            memControlGroup= new MemoryControlGroup(jtRootpath);
           
            if (isMemTrackerAvailable()) {
              LOG.info("A CGroupResourceTracker for JOB_TRACKER created.");
            }
            break;
          case TASK_TRACKER:
            String ttRootpath = conf.get(
                TaskTrackerMemoryControlGroup.CGROUP_MEM_TT_ROOT,
                TaskTrackerMemoryControlGroup.DEFAULT_TT_ROOT);
              memControlGroup = new MemoryControlGroup(ttRootpath);
             
            if (isMemTrackerAvailable()) {
              LOG.info("A CGroupResourceTracker for TASK_TRACKER created.");
            }
            break;
          case TASK:
            String rootpath = conf.get(
              TaskTrackerMemoryControlGroup.CGROUP_MEM_ROOT_PROPERTY,
              TaskTrackerMemoryControlGroup.DEFAULT_CGROUP_MEM_ROOT);
            MemoryControlGroup container = new MemoryControlGroup(rootpath);
            memControlGroup =
              container.getSubGroup(target);
           
            if (isMemTrackerAvailable()) {
              LOG.info("A CGroupResourceTracker for TASK:" +
                target + " created.");
            }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapred.ControlGroup.MemoryControlGroup

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.