Examples of JobReport


Examples of org.apache.hadoop.mapreduce.v2.api.records.JobReport

        getServiceState());
  }

  public void verifyCompleted() {
    for (Job job : getContext().getAllJobs().values()) {
      JobReport jobReport = job.getReport();
      System.out.println("Job start time :" + jobReport.getStartTime());
      System.out.println("Job finish time :" + jobReport.getFinishTime());
      Assert.assertTrue("Job start time is not less than finish time",
          jobReport.getStartTime() <= jobReport.getFinishTime());
      Assert.assertTrue("Job finish time is in future",
          jobReport.getFinishTime() <= System.currentTimeMillis());
      for (Task task : job.getTasks().values()) {
        TaskReport taskReport = task.getReport();
        System.out.println("Task start time : " + taskReport.getStartTime());
        System.out.println("Task finish time : " + taskReport.getFinishTime());
        Assert.assertTrue("Task start time is not less than finish time",
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.records.JobReport

    ret.partial = Maps.newHashMap();
    for(Map.Entry<JobId, Job> entry: mocked.entrySet()) {
      JobId id = entry.getKey();
      Job j = entry.getValue();
      ret.full.put(id, new MockCompletedJob(j));
      JobReport report = j.getReport();
      JobIndexInfo info = new JobIndexInfo(report.getStartTime(),
          report.getFinishTime(), j.getUserName(), j.getName(), id,
          j.getCompletedMaps(), j.getCompletedReduces(), String.valueOf(j.getState()));
      info.setQueueName(j.getQueueName());
      ret.partial.put(id, new PartialJob(info, id));
    }
    return ret;
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.records.JobReport

    id.setId(i);
    return id;
  }

  public static JobReport newJobReport(JobId id) {
    JobReport report = Records.newRecord(JobReport.class);
    report.setJobId(id);
    report
        .setStartTime(System.currentTimeMillis() - (int) (Math.random() * DT));
    report.setFinishTime(System.currentTimeMillis()
        + (int) (Math.random() * DT) + 1);
    report.setMapProgress((float) Math.random());
    report.setReduceProgress((float) Math.random());
    report.setJobState(JOB_STATES.next());
    return report;
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.records.JobReport

 
  public static Job newJob(ApplicationId appID, int i, int n, int m,
      Path confFile, boolean hasFailedTasks) {
    final JobId id = newJobID(appID, i);
    final String name = newJobName();
    final JobReport report = newJobReport(id);
    final Map<TaskId, Task> tasks = newTasks(id, n, m, hasFailedTasks);
    final TaskCount taskCount = getTaskCount(tasks.values());
    final Counters counters = getCounters(tasks
      .values());
    final Path configFile = confFile;

    Map<JobACL, AccessControlList> tmpJobACLs = new HashMap<JobACL, AccessControlList>();
    final Configuration conf = new Configuration();
    conf.set(JobACL.VIEW_JOB.getAclName(), "testuser");
    conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true);

    JobACLsManager aclsManager = new JobACLsManager(conf);
    tmpJobACLs = aclsManager.constructJobACLs(conf);
    final Map<JobACL, AccessControlList> jobACLs = tmpJobACLs;
    return new Job() {
      @Override
      public JobId getID() {
        return id;
      }

      @Override
      public String getName() {
        return name;
      }

      @Override
      public JobState getState() {
        return report.getJobState();
      }

      @Override
      public JobReport getReport() {
        return report;
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.records.JobReport

          try {
            LOG.info("Job end notification started for jobID : "
                + job.getReport().getJobId());
            JobEndNotifier notifier = new JobEndNotifier();
            notifier.setConf(getConfig());
            JobReport report = job.getReport();
            // If unregistration fails, the final state is unavailable. However,
            // at the last AM Retry, the client will finally be notified FAILED
            // from RM, so we should let users know FAILED via notifier as well
            if (!context.hasSuccessfullyUnregistered()) {
              report.setJobState(JobState.FAILED);
            }
            notifier.notify(report);
          } catch (InterruptedException ie) {
            LOG.warn("Job end notification interrupted for jobID : "
                + job.getReport().getJobId(), ie);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.records.JobReport

            appSubmitTime, startTime, finishTime, setupProgress, 0.0f, 0.0f,
            cleanupProgress, jobFile, amInfos, isUber, diagsb.toString());
      }

      computeProgress();
      JobReport report = MRBuilderUtils.newJobReport(jobId, jobName, username,
          state, appSubmitTime, startTime, finishTime, setupProgress,
          this.mapProgress, this.reduceProgress,
          cleanupProgress, jobFile, amInfos, isUber, diagsb.toString());
      return report;
    } finally {
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.records.JobReport

    org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
      TypeConverter.toYarn(oldJobID);
    GetJobReportRequest request =
        recordFactory.newRecordInstance(GetJobReportRequest.class);
    request.setJobId(jobId);
    JobReport report = ((GetJobReportResponse) invoke("getJobReport",
        GetJobReportRequest.class, request)).getJobReport();
    JobStatus jobStatus = null;
    if (report != null) {
      if (StringUtils.isEmpty(report.getJobFile())) {
        String jobFile = MRApps.getJobFile(conf, report.getUser(), oldJobID);
        report.setJobFile(jobFile);
      }
      String historyTrackingUrl = report.getTrackingUrl();
      String url = StringUtils.isNotEmpty(historyTrackingUrl)
          ? historyTrackingUrl : trackingUrl;
      if (!UNAVAILABLE.equals(url)) {
        url = HttpConfig.getSchemePrefix() + url;
      }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.records.JobReport

        TypeConverter.toYarn(oldJobID);
    GetJobReportRequest request =
        recordFactory.newRecordInstance(GetJobReportRequest.class);
    request.setJobId(jobId);

    JobReport report =
        ((GetJobReportResponse) invoke("getJobReport",
            GetJobReportRequest.class, request)).getJobReport();
    if (EnumSet.of(JobState.SUCCEEDED, JobState.FAILED, JobState.KILLED,
        JobState.ERROR).contains(report.getJobState())) {
      if (oldTaskAttemptID != null) {
        GetTaskAttemptReportRequest taRequest =
            recordFactory.newRecordInstance(GetTaskAttemptReportRequest.class);
        taRequest.setTaskAttemptId(TypeConverter.toYarn(oldTaskAttemptID));
        TaskAttemptReport taReport =
            ((GetTaskAttemptReportResponse) invoke("getTaskAttemptReport",
                GetTaskAttemptReportRequest.class, taRequest))
                .getTaskAttemptReport();
        if (taReport.getContainerId() == null
            || taReport.getNodeManagerHost() == null) {
          throw new IOException("Unable to get log information for task: "
              + oldTaskAttemptID);
        }
        return new LogParams(
            taReport.getContainerId().toString(),
            taReport.getContainerId().getApplicationAttemptId()
                .getApplicationId().toString(),
            NodeId.newInstance(taReport.getNodeManagerHost(),
                taReport.getNodeManagerPort()).toString(), report.getUser());
      } else {
        if (report.getAMInfos() == null || report.getAMInfos().size() == 0) {
          throw new IOException("Unable to get log information for job: "
              + oldJobID);
        }
        AMInfo amInfo = report.getAMInfos().get(report.getAMInfos().size() - 1);
        return new LogParams(
            amInfo.getContainerId().toString(),
            amInfo.getAppAttemptId().getApplicationId().toString(),
            NodeId.newInstance(amInfo.getNodeManagerHost(),
                amInfo.getNodeManagerPort()).toString(), report.getUser());
      }
    } else {
      throw new IOException("Cannot get log path for a in-progress job");
    }
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.records.JobReport

  //Check retries happen as intended
  @Test
  public void testNotifyRetries() throws InterruptedException {
    Configuration conf = new Configuration();
    conf.set(MRJobConfig.MR_JOB_END_NOTIFICATION_URL, "http://nonexistent");
    JobReport jobReport = mock(JobReport.class);
    long startTime = System.currentTimeMillis();
    this.notificationCount = 0;
    this.setConf(conf);
    this.notify(jobReport);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.records.JobReport

  }
 
  private GetJobReportResponse getJobReportResponse() {
    GetJobReportResponse jobReportResponse = Records
        .newRecord(GetJobReportResponse.class);
    JobReport jobReport = Records.newRecord(JobReport.class);
    jobReport.setJobId(jobId);
    jobReport.setJobState(JobState.SUCCEEDED);
    jobReportResponse.setJobReport(jobReport);
    return jobReportResponse;
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.