Examples of org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser

Package org.apache.hadoop.mapreduce.jobhistory

Examples of org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser

org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser
Default Parser for the JobHistory files. Typical usage is JobHistoryParser parser = new JobHistoryParser(fs, historyFile); job = parser.parse();

    FileSystem fileSys = logFile.getFileSystem(conf);
 
    // Check if the history file exists
    assertTrue("History file does not exist", fileSys.exists(logFile));


    JobHistoryParser parser = new JobHistoryParser(fileSys, 
        logFile.toUri().getPath());
    JobHistoryParser.JobInfo jobInfo = parser.parse();


    // validate format of job level key, values
    validateJobLevelKeyValuesFormat(jobInfo, status);


    // validate format of task level key, values

View Full Code Here

    FileSystem fileSys = logFile.getFileSystem(conf);
 
    // Check if the history file exists
    assertTrue("History file does not exist", fileSys.exists(logFile));


    JobHistoryParser parser = new JobHistoryParser(fileSys,
        logFile.toUri().getPath());
    
    JobHistoryParser.JobInfo jobInfo = parser.parse();
    // Now the history file contents are available in jobInfo. Let us compare
    // them with the actual values from JT.
    validateJobLevelKeyValues(mr, job, jobInfo, conf);
    validateTaskLevelKeyValues(mr, job, jobInfo);
    validateTaskAttemptLevelKeyValues(mr, job, jobInfo);

View Full Code Here

    // check history file permission
    assertTrue("History file permissions does not match", 
    fileSys.getFileStatus(logFile).getPermission().equals(
       new FsPermission(JobHistory.HISTORY_FILE_PERMISSION)));
    
    JobHistoryParser parser = new JobHistoryParser(fileSys, 
        logFile.toUri().getPath());
    JobHistoryParser.JobInfo jobInfo = parser.parse();
    


    assertTrue("Job Status read from job history file is not the expected" +
         " status", status.equals(jobInfo.getJobStatus()));
  }

View Full Code Here

    if (this.jobInfo != null) {
      return;
    }
    
    if (historyFileAbsolute != null) {
      JobHistoryParser parser = null;
      try {
        parser =
            new JobHistoryParser(historyFileAbsolute.getFileSystem(conf),
                historyFileAbsolute);
        this.jobInfo = parser.parse();
      } catch (IOException e) {
        throw new YarnRuntimeException("Could not load history file "
            + historyFileAbsolute, e);
      }
      IOException parseException = parser.getParseException(); 
      if (parseException != null) {
        throw new YarnRuntimeException(
            "Could not parse history file " + historyFileAbsolute, 
            parseException);
      }

View Full Code Here

  }


  private void parsePreviousJobHistory() throws IOException {
    FSDataInputStream in = getPreviousJobHistoryStream(getConfig(),
        appAttemptID);
    JobHistoryParser parser = new JobHistoryParser(in);
    JobInfo jobInfo = parser.parse();
    Exception parseException = parser.getParseException();
    if (parseException != null) {
      LOG.info("Got an error parsing job-history file" +
          ", ignoring incomplete events.", parseException);
    }
    Map<org.apache.hadoop.mapreduce.TaskID, TaskInfo> taskInfos = jobInfo

View Full Code Here

    } catch (IOException ioe) {
      LOG.info("Can not open history file: " + historyFilePath, ioe);
      throw (new Exception("Can not open History File"));
    }


    JobHistoryParser parser = new JobHistoryParser(in);
    JobInfo jobInfo = parser.parse();


    Assert.assertEquals("Incorrect username ", System.getProperty("user.name"),
        jobInfo.getUsername());
    Assert.assertEquals("Incorrect jobName ", "test", jobInfo.getJobname());
    Assert.assertEquals("Incorrect queuename ", "default",

View Full Code Here

        getConfig());
    //read the previous history file
    historyFile = fc.makeQualified(JobHistoryUtils.getStagingJobHistoryFile(
        histDirPath, jobName, (applicationAttemptId.getAttemptId() - 1)));          
    in = fc.open(historyFile);
    JobHistoryParser parser = new JobHistoryParser(in);
    jobInfo = parser.parse();
    Map<org.apache.hadoop.mapreduce.TaskID, TaskInfo> taskInfos = jobInfo
        .getAllTasks();
    for (TaskInfo taskInfo : taskInfos.values()) {
      if (TaskState.SUCCEEDED.toString().equals(taskInfo.getTaskStatus())) {
        completedTasks

View Full Code Here

    try {
      String historyFilePath = jtClient.getProxy().
          getJobHistoryLocationForRetiredJob(simulatedJobID);
      Path jhpath = new Path(historyFilePath);
      fs = jhpath.getFileSystem(conf);
      JobHistoryParser jhparser = new JobHistoryParser(fs, jhpath);
      JobHistoryParser.JobInfo jhInfo = jhparser.parse();
      return jhInfo;


    } finally {
      fs.close();
    }

View Full Code Here

      return; //data already loaded
    }
    
    if (historyFileAbsolute != null) {
      try {
        JobHistoryParser parser =
            new JobHistoryParser(historyFileAbsolute.getFileSystem(conf),
                historyFileAbsolute);
        jobInfo = parser.parse();
      } catch (IOException e) {
        throw new YarnException("Could not load history file "
            + historyFileAbsolute, e);
      }
    } else {

View Full Code Here

  }


  private void parse() throws IOException {
    FSDataInputStream in =
        getPreviousJobHistoryFileStream(getConfig(), applicationAttemptId);
    JobHistoryParser parser = new JobHistoryParser(in);
    jobInfo = parser.parse();
    Exception parseException = parser.getParseException();
    if (parseException != null) {
      LOG.info("Got an error parsing job-history file" + 
          ", ignoring incomplete events.", parseException);
    }
    Map<org.apache.hadoop.mapreduce.TaskID, TaskInfo> taskInfos = jobInfo

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser

org.apache.hadoop.fs.Path

org.apache.hadoop.mapred.gridmix.test.system.GridmixJobVerification

org.apache.hadoop.mapred.JSPUtil

org.apache.hadoop.mapred.TestJobHistory

org.apache.hadoop.mapred.TestJobHistoryParsing

org.apache.hadoop.mapred.TestWebUIAuthorization

org.apache.hadoop.mapreduce.TaskAttemptID

org.apache.hadoop.mapreduce.v2.app.MRAppMaster

org.apache.hadoop.mapreduce.v2.app.recover.RecoveryService

org.apache.hadoop.mapreduce.v2.hs.CompletedJob

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.