Package org.apache.hadoop.mapreduce.v2.api

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol


    //Should we compare each field or convert to V2 counter and compare
    LOG.info("CounterHS " + counterHS);
    LOG.info("CounterMR " + counterMR);
    Assert.assertEquals(counterHS, counterMR);
   
    MRClientProtocol historyClient = instantiateHistoryProxy();
    GetJobReportRequest gjReq = Records.newRecord(GetJobReportRequest.class);
    gjReq.setJobId(jobId);
    JobReport jobReport = historyClient.getJobReport(gjReq).getJobReport();
    verifyJobReport(jobReport, jobId);
  }
View Full Code Here


 
  private MRClientProtocol instantiateHistoryProxy() {
    final String serviceAddr =
        mrCluster.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS);
    final YarnRPC rpc = YarnRPC.create(conf);
    MRClientProtocol historyClient =
        (MRClientProtocol) rpc.getProxy(MRClientProtocol.class,
            NetUtils.createSocketAddr(serviceAddr), mrCluster.getConfig());
    return historyClient;
  }
View Full Code Here

    return Arrays.asList(data);
  }

  @Test
  public void testUnknownAppInRM() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(
        getJobReportResponse());
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(
        historyServerProxy, getRMDelegate());

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
View Full Code Here

  }

  @Test
  public void testRemoteExceptionFromHistoryServer() throws Exception {

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(
        RPCUtil.getRemoteException("Job ID doesnot Exist"));

    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
        .thenReturn(null);
View Full Code Here

  }

  @Test
  public void testRetriesOnConnectionFailure() throws Exception {

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(
        new RuntimeException("1")).thenThrow(new RuntimeException("2"))      
        .thenReturn(getJobReportResponse());

    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
View Full Code Here

    Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState());
  }
 
  @Test
  public void testJobReportFromHistoryServer() throws Exception {                                
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);                          
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(                     
        getJobReportResponseFromHistoryServer());                                                
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);                                    
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))                     
    .thenReturn(null);                                                                       
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(                      
View Full Code Here

    Assert.assertEquals(1.0f, jobStatus.getReduceProgress());                                    
  }
 
  @Test
  public void testCountersFromHistoryServer() throws Exception {                                
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);                          
    when(historyServerProxy.getCounters(getCountersRequest())).thenReturn(                     
        getCountersResponseFromHistoryServer());
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);                                    
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))                     
        .thenReturn(null);                                                                       
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(                      
View Full Code Here

    //as instantiateAMProxy is not called at all
    if(!isAMReachableFromClient) {
      return;
    }

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);

    // RM returns AM1 url, null, null and AM2 url on invocations.
    // Nulls simulate the time when AM2 is in the process of restarting.
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn(
        getRunningApplicationReport("am1", 78)).thenReturn(
        getRunningApplicationReport(null, 0)).thenReturn(
        getRunningApplicationReport(null, 0)).thenReturn(
        getRunningApplicationReport("am2", 90));

    GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class);
    when(jobReportResponse1.getJobReport()).thenReturn(
        MRBuilderUtils.newJobReport(jobId, "jobName-firstGen", "user",
            JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null,
            false, ""));

    // First AM returns a report with jobName firstGen and simulates AM shutdown
    // on second invocation.
    MRClientProtocol firstGenAMProxy = mock(MRClientProtocol.class);
    when(firstGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse1).thenThrow(
            new RuntimeException("AM is down!"));

    GetJobReportResponse jobReportResponse2 = mock(GetJobReportResponse.class);
    when(jobReportResponse2.getJobReport()).thenReturn(
        MRBuilderUtils.newJobReport(jobId, "jobName-secondGen", "user",
            JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null,
            false, ""));

    // Second AM generation returns a report with jobName secondGen
    MRClientProtocol secondGenAMProxy = mock(MRClientProtocol.class);
    when(secondGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse2);
   
    ClientServiceDelegate clientServiceDelegate = spy(getClientServiceDelegate(
        historyServerProxy, rmDelegate));
    // First time, connection should be to AM1, then to AM2. Further requests
View Full Code Here

    //test only applicable when AM not reachable
    if(isAMReachableFromClient) {
      return;
    }

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(                     
        getJobReportResponseFromHistoryServer());                                                

    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn(
        getRunningApplicationReport("am1", 78)).thenReturn(
View Full Code Here

    conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 3);

    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED,
        !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(getJobReportResponse());
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(
        new java.lang.reflect.UndeclaredThrowableException(new IOException(
            "Connection refuced1"))).thenThrow(
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.