Package org.apache.uima.ducc.transport.event.common

Examples of org.apache.uima.ducc.transport.event.common.IDuccSchedulingInfo


     * Update scheduler internal job structure with updated data from arriving job state.
     */
    void jobUpdate(Object state, IDuccWork job)
    {
      String methodName = "jobUpate";
        IDuccSchedulingInfo si = job.getSchedulingInfo();

        DuccId jobid = job.getDuccId();
        IRmJob j = scheduler.getJob(jobid);
        if ( j == null ) {
            // this can happen right when the job is submitted, if we haven't yet called
            // the scheduler to deal with it.  just ignore, but take note.
            // logger.info(methodName, jobid, "**** Cannot find job to update! ****");
            return;
        } else {           
            int total_work     = toInt(si.getWorkItemsTotal(), scheduler.getDefaultNTasks());
            int completed_work = toInt(si.getWorkItemsCompleted(), 0+ toInt(si.getWorkItemsError(), 0)+ toInt(si.getWorkItemsLost(), 0);

            int max_shares     = toInt(si.getSharesMax(), Integer.MAX_VALUE);
            int existing_max_shares = j.getMaxShares();

            // we need at least 1 if the job isn't reported complete, in case the user forgot to set the
            // work item count.  the job will run, but slowly in that case.
            int remaining_work = Math.max(total_work - completed_work, 1);

            double arith_mean = Double.NaN;
            IDuccPerWorkItemStatistics stats = si.getPerWorkItemStatistics();       
            if(stats != null) {
              arith_mean = stats.getMean();
            }

            logger.info(methodName, job.getDuccId(),
                        String.format("tot: %d %s -> %s compl: %s err: %s rem: %d mean: %f",
                                      total_work, 
                                      state,
                                      job.getStateObject(),
                                      si.getWorkItemsCompleted(),    // note this comes in as string (!) from OR
                                      si.getWorkItemsError(),        // also string
                                      remaining_work,
                                      arith_mean
                                      ));

            if ( max_shares != existing_max_shares ) {
                j.setMaxShares(max_shares);
                logger.info(methodName, job.getDuccId(), "Max shares adjusted from", existing_max_shares, "to", max_shares, "(incoming)",
                            si.getSharesMax());
            }
               
            j.setNQuestions(total_work, remaining_work, arith_mean);

            // formatSchedulingInfo(job.getDuccId(), si, remaining_work);
View Full Code Here


        // IRmJob j = new RmJob(job.getDuccId(), rmProps);

        // Convert Lou's structure into mine.
        IRmJob j = new RmJob(job.getDuccId());
       
        IDuccSchedulingInfo si = job.getSchedulingInfo();
        IDuccStandardInfo   sti = job.getStandardInfo();
       
        String name       = sti.getDescription();
        if ( name == null ) {
            name = "A Job With No Name.";
        }
        String user_name  = sti.getUser();
        j.setUserName(user_name);
        j.setJobName(name);

        int min_shares    = toInt(si.getSharesMin(), 0);
        int threads       = toInt(si.getThreadsPerShare(), scheduler.getDefaultNThreads());
        int user_priority = toInt(si.getSchedulingPriority(), 100);

        int total_work    =  toInt(si.getWorkItemsTotal(), scheduler.getDefaultNTasks());
        int completed_work = toInt(si.getWorkItemsCompleted(), 0);
        int remaining_work = Math.max(total_work - completed_work, 1)// never let this go 0 or negative - both cases
                                                                        // are (probably user) errors.

        logger.info(methodName, job.getDuccId(), "total_work", total_work, "completed_work", completed_work,"remaining_work", remaining_work);

        int memory        = toInt(si.getShareMemorySize(), scheduler.getDefaultMemory());
        String className  = si.getSchedulingClass();
        if ( className == null ) {
            switch ( job.getDuccType() ) {
               case Job:             
                   className = scheduler.getDefaultFairShareName();
                   break;
               case Service:
               case Pop:
               case Reservation:
                   className = scheduler.getDefaultReserveName();
                   break;
            }
            if ( className == null ) {
                j.refuse("No scheduling class defined and no default class configured.");
                return false;
            }
        }

        j.setMinShares(min_shares);
        j.setThreads(threads);
        j.setUserPriority(user_priority);
        j.setNQuestions(total_work, remaining_work, 0.0);
        j.setClassName(className);

        switch (si.getShareMemoryUnits()) {
            case GB:
                break;
            default:
                logger.warn(methodName, job.getDuccId(), "Memory units other than GB are not currently supported.  Job returned.");
                break;
        }
        j.setMemory(memory);
        j.init();

        j.setTimestamp(Long.parseLong(sti.getDateOfSubmission()));
        // logger.info(methodName, j.getId(), "SUBMISSION DATE:", subd, (new Date(subd)).toString());

        if ( job instanceof IDuccWorkJob ) {
            j.setInitWait( ((IDuccWorkJob) job).isRunnable());
        } else {
            j.setInitWait(true);                          // pop is always ready to go
        }

        j.setDuccType(job.getDuccType());                 // ugly and artificial but ... not going to rant here
                                                          // it's needed so messages can be made legible

        //
        // Now: must either create a new job, or recover one that we didn't know about, on the assumption that we
        // have just crashed and are recovering.
        //
        // Be SURE that if status is turned false for any reason, or if you exit early with false, that you
        // refuse() the job.
        //
        boolean status = true;       
       
        int max_processes = 0;
         int max_machines = 0
        ResourceClass rescl = scheduler.getResourceClass(className);
        j.setResourceClass(rescl);

        if ( rescl == null ) {
            // ph darn, we can't continue past this point
            refuse(j, "Cannot find priority class " + className + " for job");
            return false;
        }

//         if ( logger.isDebug() ) {
//             logger.debug(methodName, j.getId(),"sharesMax", si.getSharesMax());
//                        logger.debug(methodName, j.getId(),"getInstancesCount", si.getInstancesCount());
//                        logger.debug(methodName, j.getId(), "rescl.getMaxProcesses", rescl.getMaxProcesses());
//                        logger.debug(methodName, j.getId(), "rescl.getMaxMachines", rescl.getMaxMachines());
//         }

        switch ( job.getDuccType() ) {
          case Service:
          case Pop:
          case Job:             
              // instance and share count are a function of the class
              switch ( rescl.getPolicy() ) {
                  case FAIR_SHARE:
                      max_processes    = toInt(si.getSharesMax(), DEFAULT_PROCESSES);
                      max_processes    = Math.min(rescl.getMaxProcesses(), max_processes);
                      j.setMaxShares(max_processes);
                      j.setNInstances(-1);
                      break;
                     
                  case FIXED_SHARE:
                      max_processes   = toInt(si.getSharesMax(), DEFAULT_INSTANCES);
                      j.setMaxShares(max_processes);
                      j.setNInstances(max_processes);
                      break;
                     
                  case RESERVE:
                      max_machines   = toInt(si.getSharesMax(), DEFAULT_INSTANCES);
                      j.setMaxShares(max_machines);
                      j.setNInstances(max_machines);
                      break;
              }
             
              status = receiveExecutable(j, job);
              logger.trace(methodName, j.getId(), "Serivce, Pop, or Job arrives, accepted:", status);
              break;
          case Reservation:
              switch ( rescl.getPolicy() ) {
                  case FIXED_SHARE:
                      max_machines   = toInt(si.getInstancesCount(), DEFAULT_INSTANCES);
                      break;
                  case RESERVE:
                      max_machines   = toInt(si.getInstancesCount(), DEFAULT_INSTANCES);
                      break;
              }
                           
              j.setMaxShares(-1);
              j.setNInstances(max_machines);
View Full Code Here

  public String getProjection(HttpServletRequest request, IDuccWorkJob job) {
    String methodName = "getProjection";
    String retVal = "";
    try {
      WorkItemStateHelper workItemStateHelper = new WorkItemStateHelper(job);
      IDuccSchedulingInfo schedulingInfo = job.getSchedulingInfo();
      IDuccPerWorkItemStatistics perWorkItemStatistics = schedulingInfo.getPerWorkItemStatistics();
      if (perWorkItemStatistics == null) {
        return "";
      }
      int total = schedulingInfo.getIntWorkItemsTotal();
      int completed = schedulingInfo.getIntWorkItemsCompleted();
      int error = schedulingInfo.getIntWorkItemsError();
      int remainingWorkItems = total - (completed + error);
      if(remainingWorkItems > 0) {
        int usableProcessCount = job.getProcessMap().getUsableProcessCount();
        if(usableProcessCount > 0) {
          if(completed > 0) {
            int threadsPerProcess = schedulingInfo.getIntThreadsPerShare();
            int totalThreads = usableProcessCount * threadsPerProcess;
            double remainingIterations = remainingWorkItems / totalThreads;
            double avgMillis = perWorkItemStatistics.getMean();
            double leastOperatingMillis = workItemStateHelper.getLeastOperatingMillis(job);
            double mostCompletedMillis = workItemStateHelper.getMostCompletedMillis(job);
View Full Code Here

    return retVal;
  }
   
  public double getAvgMillisPerWorkItem(HttpServletRequest request, IDuccWorkJob job) {
    double avgMillis = 0;
    IDuccSchedulingInfo schedulingInfo = job.getSchedulingInfo();
    IDuccPerWorkItemStatistics perWorkItemStatistics = schedulingInfo.getPerWorkItemStatistics();
    if (perWorkItemStatistics != null) {
      avgMillis = perWorkItemStatistics.getMean();
    }
    return avgMillis;
  }
View Full Code Here

          }
        }
      }
      DuccWorkJob dwj = (DuccWorkJob) dw;
      MonitorInfo monitorInfo = mMap.get(duccId);
      IDuccSchedulingInfo si = dw.getSchedulingInfo();
      monitorInfo.total = si.getWorkItemsTotal();
      monitorInfo.done  = si.getWorkItemsCompleted();
      monitorInfo.error = ""+si.getIntWorkItemsError();
      monitorInfo.retry = si.getWorkItemsRetry();
      monitorInfo.lost = si.getWorkItemsLost();
      monitorInfo.procs = ""+dwj.getProcessMap().getAliveProcessCount();
     
      if(si.getIntWorkItemsError() > 0) {
        String logsjobdir = dwj.getUserLogsDir()+dwj.getDuccId().getFriendly()+File.separator;
        String logfile = "jd.err.log";
        ArrayList<String> errorLogs = new ArrayList<String>();
        errorLogs.add(logsjobdir+logfile);
        monitorInfo.errorLogs = errorLogs;
View Full Code Here

TOP

Related Classes of org.apache.uima.ducc.transport.event.common.IDuccSchedulingInfo

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.