long t0 = System.currentTimeMillis();
synchronized(workMap) {
Iterator<DuccId> rmResourceStateIterator = rmResourceStateMap.keySet().iterator();
while(rmResourceStateIterator.hasNext()) {
DuccId duccId = rmResourceStateIterator.next();
IRmJobState rmResourceState = rmResourceStateMap.get(duccId);
if(rmResourceState.getPendingAdditions() != null) {
logger.debug(methodName, duccId, messages.fetchLabel("pending additions")+rmResourceState.getPendingAdditions().size());
}
if(rmResourceState.getPendingRemovals() != null) {
logger.debug(methodName, duccId, messages.fetchLabel("pending removals")+rmResourceState.getPendingRemovals().size());
}
IDuccWork duccWork = workMap.findDuccWork(duccId);
if(duccWork== null) {
logger.debug(methodName, duccId, messages.fetch("not found"));
}
else {
logger.debug(methodName, duccId, messages.fetchLabel("type")+duccWork.getDuccType());
switch(duccWork.getDuccType()) {
case Job:
logger.debug(methodName, duccId, messages.fetch("processing job..."));
DuccWorkJob duccWorkJob = (DuccWorkJob) duccWork;
processPurger(duccWorkJob,rmResourceState.getResources());
changes += processMapResourcesAdd(duccWorkJob,rmResourceState.getPendingAdditions());
changes += processMapResourcesDel(duccWorkJob,rmResourceState.getPendingRemovals());
JobState jobState = duccWorkJob.getJobState();
logger.debug(methodName, duccId, messages.fetchLabel("job state")+jobState);
switch(jobState) {
case Received:
case WaitingForDriver:
logger.warn(methodName, duccId, messages.fetchLabel("unexpected state")+jobState);
break;
case WaitingForServices:
logger.debug(methodName, duccId, messages.fetchLabel("unexpected state")+jobState);
break;
case WaitingForResources:
if(rmResourceState.isRefused()) {
duccWorkJob.getStandardInfo().setDateOfCompletion(TimeStamp.getCurrentMillis());
duccWorkJob.setCompletionType(JobCompletionType.ResourcesUnavailable);
duccWorkJob.setCompletionRationale(new Rationale("resource manager refused allocation: "+rmResourceState.getReason()));
changes += stateChange(duccWorkJob,JobState.Completed);
logger.warn(methodName, duccId, messages.fetchLabel("refused")+rmResourceState.getReason());
String userName = duccWorkJob.getStandardInfo().getUser();
String userLogDir = duccWorkJob.getUserLogsDir()+duccWorkJob.getDuccId().getFriendly()+File.separator;;
String text = rmResourceState.getReason();
UserLogging.record(userName, userLogDir, text);
}
if(duccWorkJob.getProcessMap().size() > 0) {
changes += stateChange(duccWorkJob,JobState.Initializing);
logger.info(methodName, duccId, messages.fetchLabel("resources count")+duccWorkJob.getProcessMap().size());
}
break;
case Initializing:
case Running:
if(duccWorkJob.getProcessMap().size() == 0) {
changes += stateChange(duccWorkJob,JobState.WaitingForResources);
logger.info(methodName, duccId, messages.fetchLabel("resources count")+duccWorkJob.getProcessMap().size());
}
break;
case Completing:
case Completed:
logger.debug(methodName, duccId, messages.fetchLabel("unsuitable state")+jobState);
break;
case Undefined:
logger.warn(methodName, duccId, messages.fetchLabel("unsuitable state")+jobState);
break;
}
break;
case Reservation:
logger.debug(methodName, duccId, messages.fetch("processing reservation..."));
DuccWorkReservation duccWorkReservation = (DuccWorkReservation) duccWork;
changes += reservationMapResourcesAdd(duccWorkReservation,rmResourceState.getPendingAdditions());
changes += reservationMapResourcesDel(duccWorkReservation,rmResourceState.getPendingRemovals());
ReservationState reservationState = duccWorkReservation.getReservationState();
switch(reservationState) {
case Received:
logger.warn(methodName, duccId, messages.fetchLabel("unexpected state")+reservationState);
break;
case WaitingForResources:
if(rmResourceState.isRefused()) {
String schedulingClass = duccWorkReservation.getSchedulingInfo().getSchedulingClass().trim();
if(schedulingClass.equals(DuccSchedulerClasses.JobDriver)) {
if(!refusedLogged.get()) {
logger.warn(methodName, duccId, messages.fetchLabel("refusal ignored")+rmResourceState.getReason());
refusedLogged.set(true);
}
}
else {
duccWorkReservation.getStandardInfo().setDateOfCompletion(TimeStamp.getCurrentMillis());
duccWorkReservation.setCompletionType(ReservationCompletionType.ResourcesUnavailable);
duccWorkReservation.setCompletionRationale(new Rationale("resource manager refused allocation: "+rmResourceState.getReason()));
changes += stateChange(duccWorkReservation,ReservationState.Completed);
logger.warn(methodName, duccId, messages.fetchLabel("refused")+rmResourceState.getReason());
}
}
else {
if(rmResourceState.getResources() != null) {
if(!rmResourceState.getResources().isEmpty()) {
changes += stateChange(duccWorkReservation,ReservationState.Assigned);
logger.info(methodName, duccId, messages.fetchLabel("resources count")+rmResourceState.getResources().size());
}
}
else {
logger.info(methodName, duccId, messages.fetch("waiting...no resources?"));
}
}
break;
case Assigned:
if(rmResourceState.getResources() != null) {
if(rmResourceState.getResources().isEmpty()) {
changes += stateChange(duccWorkReservation,ReservationState.Completed);
logger.info(methodName, duccId, messages.fetchLabel("resources count")+rmResourceState.getResources().size());
}
}
else {
logger.info(methodName, duccId, messages.fetch("assigned...no resources?"));
}
break;
case Completed:
logger.debug(methodName, duccId, messages.fetchLabel("unsuitable state")+reservationState);
break;
case Undefined:
logger.warn(methodName, duccId, messages.fetchLabel("unsuitable state")+reservationState);
break;
}
break;
case Service:
logger.debug(methodName, duccId, messages.fetch("processing service..."));
DuccWorkJob duccWorkService = (DuccWorkJob) duccWork;
processPurger(duccWorkService,rmResourceState.getResources());
changes += processMapResourcesAdd(duccWorkService,rmResourceState.getPendingAdditions());
changes += processMapResourcesDel(duccWorkService,rmResourceState.getPendingRemovals());
JobState serviceState = duccWorkService.getJobState();
logger.debug(methodName, duccId, messages.fetchLabel("service state")+serviceState);
switch(serviceState) {
case Received:
logger.warn(methodName, duccId, messages.fetchLabel("unexpected state")+serviceState);
break;
case WaitingForServices:
logger.debug(methodName, duccId, messages.fetchLabel("unexpected state")+serviceState);
break;
case WaitingForResources:
if(rmResourceState.isRefused()) {
duccWorkService.getStandardInfo().setDateOfCompletion(TimeStamp.getCurrentMillis());
duccWorkService.setCompletionType(JobCompletionType.ResourcesUnavailable);
duccWorkService.setCompletionRationale(new Rationale("resource manager refused allocation: "+rmResourceState.getReason()));
changes += stateChange(duccWorkService,JobState.Completed);
logger.warn(methodName, duccId, messages.fetchLabel("refused")+rmResourceState.getReason());
String userName = duccWorkService.getStandardInfo().getUser();
String userLogDir = duccWorkService.getUserLogsDir()+duccWorkService.getDuccId().getFriendly()+File.separator;;
String text = rmResourceState.getReason();
UserLogging.record(userName, userLogDir, text);
}
if(duccWorkService.getProcessMap().size() > 0) {
changes += stateChange(duccWorkService,JobState.Initializing);
logger.info(methodName, duccId, messages.fetchLabel("resources count")+duccWorkService.getProcessMap().size());