processTreeInfoMap.entrySet().iterator(); it.hasNext();) {
Map.Entry<TaskAttemptID, CGroupProcessTreeInfo> entry = it.next();
TaskAttemptID attempt = entry.getKey();
CGroupProcessTreeInfo pInfo = entry.getValue();
pInfo.setMemoryUsed(0);
MemoryControlGroup taskGroup =
containerGroup.getSubGroup(attempt.toString());
if (taskGroup == null) {
LOG.warn(attempt + " has no memory CGroup");
continue;
}
long memoryUsed = taskGroup.getRSSMemoryUsage();
pInfo.setMaxMemoryUsed(memoryUsed);
memoryUsedByTasks += memoryUsed;
pInfo.setMemoryUsed(memoryUsed);
if (isTaskLimitOn && isKillable(attempt) &&
taskGroup.getMemoryUsageLimit() > 0 &&
(taskGroup.getMemoryUsageLimit() - memoryUsed
< taskLowMemoryThreshold)) {
LOG.warn(attempt + " hits the memory threshold " + memoryUsed);
tasksToKill.add(attempt);
memoryToRelease -= memoryUsed;
}
}
// kill the tasks which was at the limit
if (tasksToKill.size() > 0) {
killTasks();
taskKilled = true;
}
// check container
long containerMemoryUsed = containerGroup.getRSSMemoryUsage() + memoryUsedByTasks;
memoryLeft = containerGroup.getMemoryUsageLimit() - containerMemoryUsed;
if ((memoryLeft < lowMemoryThreshold) &&
containerGroup.getMemoryUsageLimit() > 0) {
LOG.warn("container hits the threshold for RSS memory " + containerMemoryUsed);
memoryToRelease += lowMemoryThreshold;
}
// release memory ASAP
if (memoryToRelease > 0) {
LOG.warn("Need to release " + memoryToRelease + " memory");
failTasksWithMaxMemory(memoryToRelease);
incOOMNo();
sleepTime = minMonitoringInterval;
logAliveTasks();
continue;
}
synchronized (memStat) {
memStat.setMaxMemoryUsed(containerMemoryUsed);
memStat.setMemoryUsage(containerMemoryUsed);
memStat.setMemoryLimit(containerGroup.getMemoryUsageLimit());
memStat.setLowMemoryThreshold(lowMemoryThreshold);
memStat.setMemoryUsedByTasks(memoryUsedByTasks);
long jtMemoryUsed = jtMemoryGroup.getRSSMemoryUsage();
memStat.setJTMaxMemoryUsed(jtMemoryUsed);
memStat.setJTMemoryUsage(jtMemoryUsed);
long ttMemoryUsed = ttMemoryGroup.getRSSMemoryUsage();
memStat.setTTMaxMemoryUsed(ttMemoryUsed);
memStat.setTTMemoryUsage(ttMemoryUsed);
}
sleepTime = monitoringInterval;
if (taskKilled) {
logAliveTasks();
}
// remove the very old attempts
long current = System.currentTimeMillis();
List<TaskAttemptID> oldTasks = new ArrayList<TaskAttemptID>();
oldTasks.addAll(processTreeInfoMap.keySet());
for (TaskAttemptID attempt: oldTasks) {
CGroupProcessTreeInfo pInfo = processTreeInfoMap.get(attempt);
if (pInfo == null) {
continue;
}
if (current - pInfo.getCreationTime() >= oldTaskThreshold) {
LOG.warn("Remove old entry " + attempt);
ttMemoryCGroup.removeTask(attempt.toString());
processTreeInfoMap.remove(attempt);
} else if (current - pInfo.getCreationTime() >= emptyTaskThreshold) {
MemoryControlGroup taskGroup =
containerGroup.getSubGroup(attempt.toString());
if (taskGroup == null || taskGroup.getMaxMemoryUsage() == 0) {
LOG.warn("Remove empty entry " + attempt);
processTreeInfoMap.remove(attempt);
}
}
}