Examples of JobIndexInfo

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

359
360
361
362
363
364
365
366
367
368
369
    String jobhistoryDir = JobHistoryUtils
        .getHistoryIntermediateDoneDirForUser(conf);
    JobHistory jobHistory = new JobHistory();
    jobHistory.init(conf);

    JobIndexInfo jobIndexInfo = jobHistory.getJobFileInfo(jobId)
        .getJobIndexInfo();
    String jobhistoryFileName = FileNameIndexUtils
        .getDoneFileName(jobIndexInfo);

    Path historyFilePath = new Path(jobhistoryDir, jobhistoryFileName);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

426
427
428
429
430
431
432
433
434
435
436
    String jobhistoryDir = JobHistoryUtils
        .getHistoryIntermediateDoneDirForUser(conf);
    JobHistory jobHistory = new JobHistory();
    jobHistory.init(conf);

    JobIndexInfo jobIndexInfo = jobHistory.getJobFileInfo(jobId)
        .getJobIndexInfo();
    String jobhistoryFileName = FileNameIndexUtils
        .getDoneFileName(jobIndexInfo);

    Path historyFilePath = new Path(jobhistoryDir, jobhistoryFileName);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

59
60
61
62
63
64
65
66
67
68
69
70
71
    for(Map.Entry<JobId, Job> entry: mocked.entrySet()) {
      JobId id = entry.getKey();
      Job j = entry.getValue();
      ret.full.put(id, new MockCompletedJob(j));
      JobReport report = j.getReport();
      JobIndexInfo info = new JobIndexInfo(report.getStartTime(),
          report.getFinishTime(), j.getUserName(), j.getName(), id,
          j.getCompletedMaps(), j.getCompletedReduces(), String.valueOf(j.getState()));
      info.setQueueName(j.getQueueName());
      ret.partial.put(id, new PartialJob(info, id));
    }
    return ret;
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
        doneDirFc);
    for (FileStatus fs : historyFileList) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Adding in history for " + fs.getPath());
      }
      JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
          .getName());
      String confFileName = JobHistoryUtils
          .getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName = JobHistoryUtils
          .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      HistoryFileInfo fileInfo = new HistoryFileInfo(fs.getPath(), new Path(fs
          .getPath().getParent(), confFileName), new Path(fs.getPath()
          .getParent(), summaryFileName), jobIndexInfo, true);
      jobListCache.addIfAbsent(fileInfo);
    }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
   */
  private void scanIntermediateDirectory(final Path absPath) throws IOException {
    List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(absPath,
        intermediateDoneDirFc);
    for (FileStatus fs : fileStatusList) {
      JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
          .getName());
      String confFileName = JobHistoryUtils
          .getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName = JobHistoryUtils
          .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      HistoryFileInfo fileInfo = new HistoryFileInfo(fs.getPath(), new Path(fs
          .getPath().getParent(), confFileName), new Path(fs.getPath()
          .getParent(), summaryFileName), jobIndexInfo, false);

      final HistoryFileInfo old = jobListCache.addIfAbsent(fileInfo);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
   * @throws IOException
   */
  private HistoryFileInfo getJobFileInfo(List<FileStatus> fileStatusList,
      JobId jobId) throws IOException {
    for (FileStatus fs : fileStatusList) {
      JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
          .getName());
      if (jobIndexInfo.getJobId().equals(jobId)) {
        String confFileName = JobHistoryUtils
            .getIntermediateConfFileName(jobIndexInfo.getJobId());
        String summaryFileName = JobHistoryUtils
            .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
        HistoryFileInfo fileInfo = new HistoryFileInfo(fs.getPath(), new Path(
            fs.getPath().getParent(), confFileName), new Path(fs.getPath()
            .getParent(), summaryFileName), jobIndexInfo, true);
        return fileInfo;
      }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
    Collections.sort(serialDirList);
    for (FileStatus serialDir : serialDirList) {
      List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(
          serialDir.getPath(), doneDirFc);
      for (FileStatus historyFile : historyFileList) {
        JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(historyFile
            .getPath().getName());
        long effectiveTimestamp = getEffectiveTimestamp(
            jobIndexInfo.getFinishTime(), historyFile);
        if (effectiveTimestamp <= cutoff) {
          HistoryFileInfo fileInfo = this.jobListCache.get(jobIndexInfo
              .getJobId());
          if (fileInfo == null) {
            String confFileName = JobHistoryUtils
                .getIntermediateConfFileName(jobIndexInfo.getJobId());

            fileInfo = new HistoryFileInfo(historyFile.getPath(), new Path(
                historyFile.getPath().getParent(), confFileName), null,
                jobIndexInfo, true);
          }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

754
755
756
757
758
759
760
761
762
        String jobName, JobId jobId) {
      this.historyFile = historyFile;
      this.confFile = conf;
      this.writer = writer;
      this.jobIndexInfo =
          new JobIndexInfo(-1, -1, user, jobName, jobId, -1, -1, null);
      this.jobSummary = new JobSummary();
      this.flushTimer = new Timer("FlushTimer", true);
    }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
      MockCompletedJob mockJob = new MockCompletedJob(j);
      // use MockCompletedJob to set everything below to make sure
      // consistent with what history server would do
      ret.full.put(id, mockJob);
      JobReport report = mockJob.getReport();
      JobIndexInfo info = new JobIndexInfo(report.getStartTime(),
          report.getFinishTime(), mockJob.getUserName(), mockJob.getName(), id,
          mockJob.getCompletedMaps(), mockJob.getCompletedReduces(),
          String.valueOf(mockJob.getState()));
      info.setQueueName(mockJob.getQueueName());
      ret.partial.put(id, new PartialJob(info, id));

    }
    return ret;
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

397
398
399
400
401
402
403
404
405
406
407
      String jobhistoryDir = JobHistoryUtils
          .getHistoryIntermediateDoneDirForUser(conf);
      JobHistory jobHistory = new JobHistory();
      jobHistory.init(conf);

      JobIndexInfo jobIndexInfo = jobHistory.getJobFileInfo(jobId)
          .getJobIndexInfo();
      String jobhistoryFileName = FileNameIndexUtils
          .getDoneFileName(jobIndexInfo);

      Path historyFilePath = new Path(jobhistoryDir, jobhistoryFileName);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.