Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.JobStatus

242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
    // should use the same proxy to AM2 and so instantiateProxy shouldn't be
    // called.
    doReturn(firstGenAMProxy).doReturn(secondGenAMProxy).when(
        clientServiceDelegate).instantiateAMProxy(any(String.class));

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-firstGen", jobStatus.getJobName());

    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());

    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());

    verify(clientServiceDelegate, times(2)).instantiateAMProxy(
        any(String.class));
  }
View Full Code Here
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
        getFinishedApplicationReport());

    ClientServiceDelegate clientServiceDelegate = spy(getClientServiceDelegate(
        historyServerProxy, rmDelegate));

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());
   
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(
        any(String.class));

    // Should not reach AM even for second and third times too.
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());   
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(
        any(String.class));
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());   
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(
        any(String.class));

    // The third time around, app is completed, so should go to JHS
    JobStatus jobStatus1 = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus1);
    Assert.assertEquals("TestJobFilePath", jobStatus1.getJobFile());                              
    Assert.assertEquals("http://TestTrackingUrl", jobStatus1.getTrackingUrl());                   
    Assert.assertEquals(1.0f, jobStatus1.getMapProgress());                                       
    Assert.assertEquals(1.0f, jobStatus1.getReduceProgress());
   
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(
        any(String.class));
  }
View Full Code Here
519
520
521
522
523
524
525
526
  }

  @Override
  public JobStatus getJobStatus(JobID jobID) throws IOException,
      InterruptedException {
    JobStatus status = clientCache.getClient(jobID).getJobStatus(jobID);
    return status;
  }
View Full Code Here
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
  }

  @Override
  public void killJob(JobID arg0) throws IOException, InterruptedException {
    /* check if the status is not running, if not send kill to RM */
    JobStatus status = clientCache.getClient(arg0).getJobStatus(arg0);
    if (status.getState() != JobStatus.State.RUNNING) {
      try {
        resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId());
      } catch (YarnException e) {
        throw new IOException(e);
      }
      return;
    }

    try {
      /* send a kill to the AM */
      clientCache.getClient(arg0).killJob(arg0);
      long currentTimeMillis = System.currentTimeMillis();
      long timeKillIssued = currentTimeMillis;
      while ((currentTimeMillis < timeKillIssued + 10000L) && (status.getState()
          != JobStatus.State.KILLED)) {
          try {
            Thread.sleep(1000L);
          } catch(InterruptedException ie) {
            /** interrupted, just break */
            break;
          }
          currentTimeMillis = System.currentTimeMillis();
          status = clientCache.getClient(arg0).getJobStatus(arg0);
      }
    } catch(IOException io) {
      LOG.debug("Error when checking for application status", io);
    }
    if (status.getState() != JobStatus.State.KILLED) {
      try {
        resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId());
      } catch (YarnException e) {
        throw new IOException(e);
      }
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
        Job hadoopRunningJob = jobControl.getRunningJob();
        if (hadoopRunningJob != null) {
            JobID hadoopJobId = hadoopRunningJob.getJobID();
            logger.debug("found active hadoop job:", hadoopJobId.toString());

            JobStatus status = hadoopRunningJob.getStatus();
            float progress = 0.25f * (
                    status.getSetupProgress() +
                            status.getMapProgress() +
                            status.getReduceProgress() +
                            status.getCleanupProgress());

            logger.debug("found progress: "
                    + status.getSetupProgress() + " "
                    + status.getMapProgress() + " "
                    + status.getReduceProgress() + " "
                    + status.getCleanupProgress() + " "
                    + progress);

            if (jobMap.containsKey(hadoopJobId)) {
                setJobProgress(jobMap.get(hadoopJobId), progress);
            } else {
View Full Code Here
697
698
699
700
701
702
703
704
705
706
707
708
709
  }

  @Override
  public void killJob(JobID arg0) throws IOException, InterruptedException {
    /* check if the status is not running, if not send kill to RM */
    JobStatus status = getJobStatus(arg0);
    if (status.getState() == JobStatus.State.RUNNING ||
        status.getState() == JobStatus.State.PREP) {
      try {
        resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId());
      } catch (YarnException e) {
        throw new IOException(e);
      }
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
    long startTime = System.currentTimeMillis();

    JobID jobID = new JobID(String.valueOf(startTime), 12345);

    JobStatus mockJobStatus = mock(JobStatus.class);
    when(mockJobStatus.getJobID()).thenReturn(jobID);
    when(mockJobStatus.getState()).thenReturn(JobStatus.State.RUNNING);
    when(mockJobStatus.getStartTime()).thenReturn(startTime);
    when(mockJobStatus.getUsername()).thenReturn("mockuser");
    when(mockJobStatus.getQueue()).thenReturn("mockqueue");
    when(mockJobStatus.getPriority()).thenReturn(JobPriority.NORMAL);
    when(mockJobStatus.getNumUsedSlots()).thenReturn(1);
    when(mockJobStatus.getNumReservedSlots()).thenReturn(1);
    when(mockJobStatus.getUsedMem()).thenReturn(1024);
    when(mockJobStatus.getReservedMem()).thenReturn(512);
    when(mockJobStatus.getNeededMem()).thenReturn(2048);
    when(mockJobStatus.getSchedulingInfo()).thenReturn("NA");

    Job mockJob = mock(Job.class);
    when(mockJob.getTaskReports(isA(TaskType.class))).thenReturn(
      new TaskReport[5]);
View Full Code Here
91
92
93
94
95
96
97
98
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(
        getJobReportResponse());
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(
        historyServerProxy, getRMDelegate());

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
  }
View Full Code Here
132
133
134
135
136
137
138
139
140
141
        .thenReturn(null);

    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(
        historyServerProxy, rm);

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    verify(historyServerProxy, times(3)).getJobReport(
        any(GetJobReportRequest.class));
  }
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
  @Test
  public void testHistoryServerNotConfigured() throws Exception {
    //RM doesn't have app report and job History Server is not configured
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(
        null, getRMDelegate());
    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertEquals("N/A", jobStatus.getUsername());
    Assert.assertEquals(JobStatus.State.PREP, jobStatus.getState());

    //RM has app report and job History Server is not configured
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    ApplicationReport applicationReport = getFinishedApplicationReport();
    when(rm.getApplicationReport(jobId.getAppId())).thenReturn(
        applicationReport);

    clientServiceDelegate = getClientServiceDelegate(null, rm);
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertEquals(applicationReport.getUser(), jobStatus.getUsername());
    Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState());
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.JobStatus

  • org.apache.hadoop.mapred.ClientServiceDelegate
  • org.apache.hadoop.mapred.gridmix.JobMonitor$MonitorThread
  • org.apache.hadoop.mapred.gridmix.StressJobFactory
  • org.apache.hadoop.mapred.gridmix.TestGridmixStatistics
  • org.apache.hadoop.mapred.JobClientUnitTest
  • org.apache.hadoop.mapred.MockSimulatorJobTracker
  • org.apache.hadoop.mapred.SimulatorJobClient
  • org.apache.hadoop.mapred.TestClientServiceDelegate
  • org.apache.hadoop.mapred.YARNRunner
  • org.apache.hadoop.security.authorize.AccessControlList
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.