Examples of IRmJob


Examples of org.apache.uima.ducc.rm.scheduler.IRmJob

        //     rmProps.setProperty(rmProperties[i], v);
        // }
        // IRmJob j = new RmJob(job.getDuccId(), rmProps);

        // Convert Lou's structure into mine.
        IRmJob j = new RmJob(job.getDuccId());
       
        IDuccSchedulingInfo si = job.getSchedulingInfo();
        IDuccStandardInfo   sti = job.getStandardInfo();
       
        String name       = sti.getDescription();
        if ( name == null ) {
            name = "A Job With No Name.";
        }
        String user_name  = sti.getUser();
        j.setUserName(user_name);
        j.setJobName(name);

        int min_shares    = toInt(si.getSharesMin(), 0);
        int threads       = toInt(si.getThreadsPerShare(), scheduler.getDefaultNThreads());
        int user_priority = toInt(si.getSchedulingPriority(), 100);

        int total_work    =  toInt(si.getWorkItemsTotal(), scheduler.getDefaultNTasks());
        int completed_work = toInt(si.getWorkItemsCompleted(), 0);
        int remaining_work = Math.max(total_work - completed_work, 1)// never let this go 0 or negative - both cases
                                                                        // are (probably user) errors.

        logger.info(methodName, job.getDuccId(), "total_work", total_work, "completed_work", completed_work,"remaining_work", remaining_work);

        int memory        = toInt(si.getShareMemorySize(), scheduler.getDefaultMemory());
        String className  = si.getSchedulingClass();
        if ( className == null ) {
            switch ( job.getDuccType() ) {
               case Job:             
                   className = scheduler.getDefaultFairShareName();
                   break;
               case Service:
               case Pop:
               case Reservation:
                   className = scheduler.getDefaultReserveName();
                   break;
            }
            if ( className == null ) {
                j.refuse("No scheduling class defined and no default class configured.");
                return false;
            }
        }

        j.setMinShares(min_shares);
        j.setThreads(threads);
        j.setUserPriority(user_priority);
        j.setNQuestions(total_work, remaining_work, 0.0);
        j.setClassName(className);

        switch (si.getShareMemoryUnits()) {
            case GB:
                break;
            default:
                logger.warn(methodName, job.getDuccId(), "Memory units other than GB are not currently supported.  Job returned.");
                break;
        }
        j.setMemory(memory);
        j.init();

        j.setTimestamp(Long.parseLong(sti.getDateOfSubmission()));
        // logger.info(methodName, j.getId(), "SUBMISSION DATE:", subd, (new Date(subd)).toString());

        if ( job instanceof IDuccWorkJob ) {
            j.setInitWait( ((IDuccWorkJob) job).isRunnable());
        } else {
            j.setInitWait(true);                          // pop is always ready to go
        }

        j.setDuccType(job.getDuccType());                 // ugly and artificial but ... not going to rant here
                                                          // it's needed so messages can be made legible

        //
        // Now: must either create a new job, or recover one that we didn't know about, on the assumption that we
        // have just crashed and are recovering.
        //
        // Be SURE that if status is turned false for any reason, or if you exit early with false, that you
        // refuse() the job.
        //
        boolean status = true;       
       
        int max_processes = 0;
         int max_machines = 0
        ResourceClass rescl = scheduler.getResourceClass(className);
        j.setResourceClass(rescl);

        if ( rescl == null ) {
            // ph darn, we can't continue past this point
            refuse(j, "Cannot find priority class " + className + " for job");
            return false;
        }

//         if ( logger.isDebug() ) {
//             logger.debug(methodName, j.getId(),"sharesMax", si.getSharesMax());
//                        logger.debug(methodName, j.getId(),"getInstancesCount", si.getInstancesCount());
//                        logger.debug(methodName, j.getId(), "rescl.getMaxProcesses", rescl.getMaxProcesses());
//                        logger.debug(methodName, j.getId(), "rescl.getMaxMachines", rescl.getMaxMachines());
//         }

        switch ( job.getDuccType() ) {
          case Service:
          case Pop:
          case Job:             
              // instance and share count are a function of the class
              switch ( rescl.getPolicy() ) {
                  case FAIR_SHARE:
                      max_processes    = toInt(si.getSharesMax(), DEFAULT_PROCESSES);
                      max_processes    = Math.min(rescl.getMaxProcesses(), max_processes);
                      j.setMaxShares(max_processes);
                      j.setNInstances(-1);
                      break;
                     
                  case FIXED_SHARE:
                      max_processes   = toInt(si.getSharesMax(), DEFAULT_INSTANCES);
                      j.setMaxShares(max_processes);
                      j.setNInstances(max_processes);
                      break;
                     
                  case RESERVE:
                      max_machines   = toInt(si.getSharesMax(), DEFAULT_INSTANCES);
                      j.setMaxShares(max_machines);
                      j.setNInstances(max_machines);
                      break;
              }
             
              status = receiveExecutable(j, job);
              logger.trace(methodName, j.getId(), "Serivce, Pop, or Job arrives, accepted:", status);
              break;
          case Reservation:
              switch ( rescl.getPolicy() ) {
                  case FIXED_SHARE:
                      max_machines   = toInt(si.getInstancesCount(), DEFAULT_INSTANCES);
                      break;
                  case RESERVE:
                      max_machines   = toInt(si.getInstancesCount(), DEFAULT_INSTANCES);
                      break;
              }
                           
              j.setMaxShares(-1);
              j.setNInstances(max_machines);

              status = receiveReservation(j, job);
              logger.trace(methodName, j.getId(), "Reservation arrives, accepted:", status);
              break;
          default:
              refuse(j, "Unknown job type: " + job.getDuccType());
              status = false;
              break;
View Full Code Here

Examples of org.apache.uima.ducc.rm.scheduler.IRmJob

        // gone stuff in in the right side of the map
        Map<DuccId, IDuccProcess> rproc = diffmap.getRight();
        for ( IDuccProcess p : rproc .values()) {
            // these processes are done.  look up the job and tell it process complete.
            Share s = scheduler.getShare(p.getDuccId());
            IRmJob j = scheduler.getJob(jobid);
            if ( j == null ) {
                throw new SchedulingException(jobid, "Process completion arrives for share " + s.toString() +
                                              " but job " + jobid + "cannot be found.");
            }
            scheduler.signalCompletion(j, s);
            logger.info(methodName, jobid,
                         String.format("Process %5s", p.getPID()),
                         "Completion:", s.toString());
        }

        for( DuccMapValueDifference<IDuccProcess> pd: diffmap ) {
            IDuccProcess pl = pd.getLeft();
            IDuccProcess pr = pd.getRight();

            Share sl = scheduler.getShare(pl.getDuccId());
            Share sr = scheduler.getShare(pr.getDuccId());

            String shareL = ( sl == null ) ? "<none>" : sl.toString();
            String shareR = ( sr == null ) ? "<none>" : sr.toString();

            ITimeWindow initL = pl.getTimeWindowInit();
            ITimeWindow initR = pr.getTimeWindowInit();
            long init_timeL = (initL == null) ? 0 : initL.getElapsedMillis();
            long init_timeR = (initR == null) ? 0 : initR.getElapsedMillis();

            /** extreme debugging only*/
            if ( logger.isTrace() ) {
                logger.trace(methodName, jobid,
                             "\n\tReconciling. incoming.(pid, mem, state, share, initTime)",
                             pl.getPID(),
                             pl.getResidentMemory(),
                             pl.getProcessState(),
                             shareL,
                             init_timeL,
                             "\n\tReconciling. existing.(pid, mem, state, share, initTime)",
                             pr.getPID(),
                             pr.getResidentMemory(),
                             pr.getProcessState(),
                             shareR,
                             init_timeR
                             );
            } else {
                if ( (pr.getPID() == null) && (pl.getPID() != null) ) {
                    logger.trace(methodName, jobid,
                                String.format("Process %5s", pl.getPID()),
                                "PID assignement for share", shareL);
                }
                if ( pl.getProcessState() != pr.getProcessState() ) {
                    logger.info(methodName, jobid,
                                String.format("Process %5s", pl.getPID()), sl.toString(),
                                "State:", pr.getProcessState(), "->", pl.getProcessState(),
                                getElapsedTime(pr.getTimeWindowInit()), getElapsedTime(pr.getTimeWindowRun()));
                }
            }

            long mem = pl.getResidentMemory();
            ProcessState state = pl.getProcessState();
            String pid = pl.getPID();                       
            Share s = scheduler.getShare(pl.getDuccId());
            if ( pl.isActive() ) {
               
                if ( s == null ) {
                    // this can happen if a node dies and the share is purged so it's ok.
                    logger.warn(methodName, jobid, "Update for share from process", pl.getPID(), pl.getDuccId(), "but cannot find share.");
                    continue;
                }
               
                if ( s.isPurged() ) {
                    IRmJob j = scheduler.getJob(jobid);
                    scheduler.signalCompletion(j, s);
                    logger.info(methodName, jobid, "Process", pl.getPID(), "marked complete because it is purged. State:", state);
                }

                if ( ! s.update(jobid, mem, state, pl.getTimeWindowInit(), pl.getTimeWindowRun(), pid) ) {
                    // TODO: probably change to just a warning and cancel the job - for now I want an attention-getter
                    throw new SchedulingException(jobid, "Process update arrives for share " + s.toString() +
                                                  " but jobid " + jobid + " does not match job in share " + s.getJob().getId());
                }
                // logger.debug(methodName, jobid, "Process update to process ", pid, "mem", mem, "state", state, "is assigned for share", s.toString());

            } else if ( pl.isComplete() ) {
                if ( s != null ) {              // in some final states the share is already gone, not an error (e.g. Stopped)
                    IRmJob j = scheduler.getJob(jobid);
                    scheduler.signalCompletion(j, s);
                    logger.info(methodName, jobid, "Process", pl.getPID(), " completed due to state", state);
                }
            } else {
                logger.info(methodName, jobid, "Process", pl.getPID(), "ignoring update because of state", state);
View Full Code Here

Examples of org.apache.uima.ducc.rm.scheduler.IRmJob

    {
      String methodName = "jobUpate";
        IDuccSchedulingInfo si = job.getSchedulingInfo();

        DuccId jobid = job.getDuccId();
        IRmJob j = scheduler.getJob(jobid);
        if ( j == null ) {
            // this can happen right when the job is submitted, if we haven't yet called
            // the scheduler to deal with it.  just ignore, but take note.
            // logger.info(methodName, jobid, "**** Cannot find job to update! ****");
            return;
        } else {           
            int total_work     = toInt(si.getWorkItemsTotal(), scheduler.getDefaultNTasks());
            int completed_work = toInt(si.getWorkItemsCompleted(), 0+ toInt(si.getWorkItemsError(), 0)+ toInt(si.getWorkItemsLost(), 0);

            int max_shares     = toInt(si.getSharesMax(), Integer.MAX_VALUE);
            int existing_max_shares = j.getMaxShares();

            // we need at least 1 if the job isn't reported complete, in case the user forgot to set the
            // work item count.  the job will run, but slowly in that case.
            int remaining_work = Math.max(total_work - completed_work, 1);

            double arith_mean = Double.NaN;
            IDuccPerWorkItemStatistics stats = si.getPerWorkItemStatistics();       
            if(stats != null) {
              arith_mean = stats.getMean();
            }

            logger.info(methodName, job.getDuccId(),
                        String.format("tot: %d %s -> %s compl: %s err: %s rem: %d mean: %f",
                                      total_work, 
                                      state,
                                      job.getStateObject(),
                                      si.getWorkItemsCompleted(),    // note this comes in as string (!) from OR
                                      si.getWorkItemsError(),        // also string
                                      remaining_work,
                                      arith_mean
                                      ));

            if ( max_shares != existing_max_shares ) {
                j.setMaxShares(max_shares);
                logger.info(methodName, job.getDuccId(), "Max shares adjusted from", existing_max_shares, "to", max_shares, "(incoming)",
                            si.getSharesMax());
            }
               
            j.setNQuestions(total_work, remaining_work, arith_mean);

            // formatSchedulingInfo(job.getDuccId(), si, remaining_work);

            if ( job instanceof IDuccWorkJob ) {
                if ( j.setInitWait( ((IDuccWorkJob) job).isRunnable()) ) {
                    logger.info(methodName, jobid, "Set Initialized.");
                    scheduler.signalInitialized(j);
                }
            } else {
                j.setInitWait(true);                           // pop is always ready to go
            }           
        }
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.