Examples of org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo

Package org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser

Examples of org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo

org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo

      LOG.info("Can not open history file: " + historyFilePath, ioe);
      throw (new Exception("Can not open History File"));
    }


    JobHistoryParser parser = new JobHistoryParser(in);
    JobInfo jobInfo = parser.parse();
    Exception parseException = parser.getParseException();
    Assert.assertNull("Caught an expected exception " + parseException, 
        parseException);
    int noOffailedAttempts = 0;
    Map<TaskID, TaskInfo> allTasks = jobInfo.getAllTasks();
    for (Task task : job.getTasks().values()) {
      TaskInfo taskInfo = allTasks.get(TypeConverter.fromYarn(task.getID()));
      for (TaskAttempt taskAttempt : task.getAttempts().values()) {
        TaskAttemptInfo taskAttemptInfo = taskInfo.getAllTaskAttempts().get(
            TypeConverter.fromYarn((taskAttempt.getID())));

View Full Code Here

      LOG.info("Can not open history file: " + historyFilePath, ioe);
      throw (new Exception("Can not open History File"));
    }


    JobHistoryParser parser = new JobHistoryParser(in);
    JobInfo jobInfo = parser.parse();
    Exception parseException = parser.getParseException();
    Assert.assertNull("Caught an expected exception " + parseException,
        parseException);
    for (Map.Entry<TaskID,TaskInfo> entry : jobInfo.getAllTasks().entrySet()) {
      TaskId yarnTaskID = TypeConverter.toYarn(entry.getKey());
      CompletedTask ct = new CompletedTask(yarnTaskID, entry.getValue());
      Assert.assertNotNull("completed task report has null counters",
          ct.getReport().getCounters());
    }

View Full Code Here

   */
  static JobInfo getJobInfo(Path logFile, FileSystem fs,
      JobTracker jobTracker) throws IOException {
    String jobid =
        JobHistory.getJobIDFromHistoryFilePath(logFile).toString();
    JobInfo jobInfo = null;
    synchronized(jobHistoryCache) {
      jobInfo = jobHistoryCache.remove(jobid);
      if (jobInfo == null) {
        JobHistoryParser parser = new JobHistoryParser(fs, logFile);
        jobInfo = parser.parse();

View Full Code Here

      InterruptedException, ServletException {
    String jobid =
        JobHistory.getJobIDFromHistoryFilePath(logFile).toString();
    String user = request.getRemoteUser();


    JobInfo jobInfo = JSPUtil.getJobInfo(logFile, fs, jobTracker);
    if (user != null) {
      // authorize user for job-view access
      try {
        final UserGroupInformation ugi =
            UserGroupInformation.createRemoteUser(user);
        
        AccessControlList viewJobAcl = jobInfo.getJobACLs().get(JobACL.VIEW_JOB);
        if (viewJobAcl == null) {
          // may be older job history file of earlier unsecure cluster
          viewJobAcl = new AccessControlList("*");
        }


        jobTracker.getACLsManager().checkAccess(jobid, ugi,
            jobInfo.getJobQueueName(), Operation.VIEW_JOB_DETAILS,
            jobInfo.getUsername(), viewJobAcl);


      } catch (AccessControlException e) {
        String errMsg =
            String.format(
                "User %s failed to view %s!<br><br>%s"

View Full Code Here

    final Path jobFile = new Path(logFile);
    String jobid = JobHistory.getJobIDFromHistoryFilePath(jobFile).toString();


    final FileSystem fs = (FileSystem) application.getAttribute("fileSys");
    final JobTracker jobTracker = (JobTracker) application.getAttribute("job.tracker");
    JobInfo job = JSPUtil.checkAccessAndGetJobInfo(request, response,
        jobTracker, fs, jobFile);
    if (job == null) {
      return;
    }


      out.write("\n\n<html>\n<head>\n<title>Hadoop Job ");
      out.print(jobid);
      out.write(" on History Viewer</title>\n<link rel=\"stylesheet\" type=\"text/css\" href=\"/static/hadoop.css\">\n</head>\n<body>\n\n<h2>Hadoop Job ");
      out.print(jobid );
      out.write(" on <a href=\"jobhistory.jsp\">History Viewer</a></h2>\n\n<b>User: </b> ");
      out.print(HtmlQuoting.quoteHtmlChars(job.getUsername()) );
      out.write("<br/>\n<b>JobName: </b> ");
      out.print(HtmlQuoting.quoteHtmlChars(job.getJobname()) );
      out.write("<br/>\n<b>JobConf: </b> <a href=\"jobconf_history.jsp?logFile=");
      out.print(logFile);
      out.write("\"> \n                 ");
      out.print(job.getJobConfPath() );
      out.write("</a><br/> \n");
         
  Map<JobACL, AccessControlList> jobAcls = job.getJobACLs();
  JSPUtil.printJobACLs(jobTracker, jobAcls, out);


      out.write("\n<b>Submitted At: </b> ");
      out.print(StringUtils.getFormattedTimeWithDiff(dateFormat, job.getSubmitTime(), 0 )  );
      out.write("<br/> \n<b>Launched At: </b> ");
      out.print(StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLaunchTime(), job.getSubmitTime()) );
      out.write("<br/>\n<b>Finished At: </b>  ");
      out.print(StringUtils.getFormattedTimeWithDiff(dateFormat, job.getFinishTime(), job.getLaunchTime()) );
      out.write("<br/>\n<b>Status: </b> ");
      out.print( ((job.getJobStatus()) == null ? "Incomplete" :job.getJobStatus()) );
      out.write("<br/> \n");


    HistoryViewer.SummarizedJob sj = new HistoryViewer.SummarizedJob(job);


      out.write("\n<b><a href=\"analysejobhistory.jsp?logFile=");
      out.print(logFile);
      out.write("\">Analyse This Job</a></b> \n<hr/>\n<center>\n<table border=\"2\" cellpadding=\"5\" cellspacing=\"2\">\n<tr>\n<td>Kind</td><td>Total Tasks(successful+failed+killed)</td><td>Successful tasks</td><td>Failed tasks</td><td>Killed tasks</td><td>Start Time</td><td>Finish Time</td>\n</tr>\n<tr>\n<td>Setup</td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=JOB_SETUP&status=all\">\n        ");
      out.print(sj.getTotalSetups());
      out.write("</a></td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=JOB_SETUP&status=SUCCEEDED\">\n        ");
      out.print(sj.getNumFinishedSetups());
      out.write("</a></td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=JOB_SETUP&status=FAILED\">\n        ");
      out.print(sj.getNumFailedSetups());
      out.write("</a></td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=JOB_SETUP&status=KILLED\">\n        ");
      out.print(sj.getNumKilledSetups());
      out.write("</a></td>  \n    <td>");
      out.print(StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getSetupStarted(), 0) );
      out.write("</td>\n    <td>");
      out.print(StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getSetupFinished(), sj.getSetupStarted()) );
      out.write("</td>\n</tr>\n<tr>\n<td>Map</td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=MAP&status=all\">\n        ");
      out.print(sj.getTotalMaps());
      out.write("</a></td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=MAP&status=SUCCEEDED\">\n        ");
      out.print(job.getFinishedMaps() );
      out.write("</a></td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=MAP&status=FAILED\">\n        ");
      out.print(sj.getNumFailedMaps());
      out.write("</a></td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=MAP&status=KILLED\">\n        ");
      out.print(sj.getNumKilledMaps());
      out.write("</a></td>\n    <td>");
      out.print(StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getMapStarted(), 0) );
      out.write("</td>\n    <td>");
      out.print(StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getMapFinished(), sj.getMapStarted()) );
      out.write("</td>\n</tr>\n<tr>\n<td>Reduce</td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=REDUCE&status=all\">\n        ");
      out.print(sj.getTotalReduces());
      out.write("</a></td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=REDUCE&status=SUCCEEDED\">\n        ");
      out.print(job.getFinishedReduces());
      out.write("</a></td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=REDUCE&status=FAILED\">\n        ");
      out.print(sj.getNumFailedReduces());
      out.write("</a></td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=REDUCE&status=KILLED\">\n        ");
      out.print(sj.getNumKilledReduces());
      out.write("</a></td>  \n    <td>");
      out.print(StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceStarted(), 0) );
      out.write("</td>\n    <td>");
      out.print(StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceFinished(), sj.getReduceStarted()) );
      out.write("</td>\n</tr>\n<tr>\n<td>Cleanup</td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=JOB_CLEANUP&status=all\">\n        ");
      out.print(sj.getTotalCleanups());
      out.write("</a></td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=JOB_CLEANUP&status=SUCCEEDED\">\n        ");
      out.print(sj.getNumFinishedCleanups());
      out.write("</a></td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=JOB_CLEANUP&status=FAILED\">\n        ");
      out.print(sj.getNumFailedCleanups());
      out.write("</a></td>\n    <td><a href=\"jobtaskshistory.jsp?logFile=");
      out.print(logFile);
      out.write("&taskType=JOB_CLEANUP&status=KILLED>\">\n        ");
      out.print(sj.getNumKilledCleanups());
      out.write("</a></td>  \n    <td>");
      out.print(StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getCleanupStarted(), 0) );
      out.write("</td>\n    <td>");
      out.print(StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getCleanupFinished(), sj.getCleanupStarted()) );
      out.write("</td>\n</tr>\n</table>\n\n<br>\n<br>\n\n<table border=2 cellpadding=\"5\" cellspacing=\"2\">\n  <tr>\n  <th><br/></th>\n  <th>Counter</th>\n  <th>Map</th>\n  <th>Reduce</th>\n  <th>Total</th>\n</tr>\n\n");
  


 Counters totalCounters = job.getTotalCounters();
 Counters mapCounters = job.getMapCounters();
 Counters reduceCounters = job.getReduceCounters();


 if (totalCounters != null) {
   for (String groupName : totalCounters.getGroupNames()) {
     CounterGroup totalGroup = totalCounters.getGroup(groupName);
     CounterGroup mapGroup = mapCounters.getGroup(groupName);

View Full Code Here

    
    /*
     * Read the job information necessary for post performance analysis
     */
    JobConf jobConf = new JobConf();
    JobInfo jobInfo = readJobInformation(jobConf);
    this._jobExecutionStatistics = new JobStatistics(jobConf, jobInfo);
  }

View Full Code Here

    
    /* 
     * Read JobHistoryFile and build job counters to evaluate diagnostic rules
     */
    JobHistoryParser parser;
    JobInfo jobInfo;
    if (jobHistoryFileUrl.getProtocol().equals("hdfs")) {
      parser = new JobHistoryParser(FileSystem.get(jobConf),
                                    jobHistoryFileUrl.getPath());
      jobInfo = parser.parse();
    } else if (jobHistoryFileUrl.getProtocol().equals("file")) {

View Full Code Here

    }


    JobHistoryParser parser =
        new JobHistoryParser(new Path(historyFileName).getFileSystem(conf),
            historyFileName);
    JobInfo jobInfo = parser.parse();
    Map<TaskID, TaskInfo> tipsMap = jobInfo.getAllTasks();
    for (TaskID tip : tipsMap.keySet()) {
      // validate access of taskdetailshistory.jsp
      validateViewJob(jtURL + "/taskdetailshistory.jsp?logFile="
          + historyFileName + "&tipid=" + tip.toString(), "GET");

View Full Code Here

    }
  }


  @Test (timeout=50000)
  public void testJobInfo() throws Exception {
    JobInfo info = new JobInfo();
    Assert.assertEquals("NORMAL", info.getPriority());
    info.printAll();
  }

View Full Code Here

    }


    JobHistory jobHistory = new JobHistory();
    jobHistory.init(conf);
    HistoryFileInfo fileInfo = jobHistory.getJobFileInfo(jobId);
    JobInfo jobInfo;
    long numFinishedMaps;
    
    synchronized(fileInfo) {
      Path historyFilePath = fileInfo.getHistoryFile();
      FSDataInputStream in = null;
      LOG.info("JobHistoryFile is: " + historyFilePath);
      try {
        in = fc.open(fc.makeQualified(historyFilePath));
      } catch (IOException ioe) {
        LOG.info("Can not open history file: " + historyFilePath, ioe);
        throw (new Exception("Can not open History File"));
      }


      JobHistoryParser parser = new JobHistoryParser(in);
      final EventReader realReader = new EventReader(in);
      EventReader reader = Mockito.mock(EventReader.class);
      if (numMaps == numSuccessfulMaps) {
        reader = realReader;
      } else {
        final AtomicInteger numFinishedEvents = new AtomicInteger(0);  // Hack!
        Mockito.when(reader.getNextEvent()).thenAnswer(
            new Answer<HistoryEvent>() {
              public HistoryEvent answer(InvocationOnMock invocation) 
              throws IOException {
                HistoryEvent event = realReader.getNextEvent();
                if (event instanceof TaskFinishedEvent) {
                  numFinishedEvents.incrementAndGet();
                }


                if (numFinishedEvents.get() <= numSuccessfulMaps) {
                  return event;
                } else {
                  throw new IOException("test");
                }
              }
            }
        );
      }


      jobInfo = parser.parse(reader);


      numFinishedMaps = 
        computeFinishedMaps(jobInfo, numMaps, numSuccessfulMaps);


      if (numFinishedMaps != numMaps) {
        Exception parseException = parser.getParseException();
        Assert.assertNotNull("Didn't get expected parse exception", 
            parseException);
      }
    }
    
    Assert.assertEquals("Incorrect username ", System.getProperty("user.name"),
        jobInfo.getUsername());
    Assert.assertEquals("Incorrect jobName ", "test", jobInfo.getJobname());
    Assert.assertEquals("Incorrect queuename ", "default",
        jobInfo.getJobQueueName());
    Assert
        .assertEquals("incorrect conf path", "test", jobInfo.getJobConfPath());
    Assert.assertEquals("incorrect finishedMap ", numSuccessfulMaps, 
        numFinishedMaps);
    Assert.assertEquals("incorrect finishedReduces ", numReduces,
        jobInfo.getFinishedReduces());
    Assert.assertEquals("incorrect uberized ", job.isUber(),
        jobInfo.getUberized());
    Map<TaskID, TaskInfo> allTasks = jobInfo.getAllTasks();
    int totalTasks = allTasks.size();
    Assert.assertEquals("total number of tasks is incorrect  ", 
        (numMaps+numReduces), totalTasks);


    // Verify aminfo
    Assert.assertEquals(1, jobInfo.getAMInfos().size());
    Assert.assertEquals(MRApp.NM_HOST, jobInfo.getAMInfos().get(0)
        .getNodeManagerHost());
    AMInfo amInfo = jobInfo.getAMInfos().get(0);
    Assert.assertEquals(MRApp.NM_PORT, amInfo.getNodeManagerPort());
    Assert.assertEquals(MRApp.NM_HTTP_PORT, amInfo.getNodeManagerHttpPort());
    Assert.assertEquals(1, amInfo.getAppAttemptId().getAttemptId());
    Assert.assertEquals(amInfo.getAppAttemptId(), amInfo.getContainerId()
        .getApplicationAttemptId());

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo

org.apache.hadoop.mapred.jobdetailshistory_jsp

org.apache.hadoop.mapred.JSPUtil

org.apache.hadoop.mapred.TestWebUIAuthorization

org.apache.hadoop.mapreduce.v2.app.MRAppMaster

org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryParsing

org.apache.hadoop.vaidya.postexdiagnosis.PostExPerformanceDiagnoser

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.