Examples of MRClientProtocol


Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

  }

  @Test
  public void testRetriesOnConnectionFailure() throws Exception {

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(
        new RuntimeException("1")).thenThrow(new RuntimeException("2"))      
        .thenReturn(getJobReportResponse());

    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState());
  }
 
  @Test
  public void testJobReportFromHistoryServer() throws Exception {                                
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);                          
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(                     
        getJobReportResponseFromHistoryServer());                                                
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);                                    
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))                     
    .thenReturn(null);                                                                       
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(                      
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    Assert.assertEquals(1.0f, jobStatus.getReduceProgress());                                    
  }
 
  @Test
  public void testCountersFromHistoryServer() throws Exception {                                
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);                          
    when(historyServerProxy.getCounters(getCountersRequest())).thenReturn(                     
        getCountersResponseFromHistoryServer());
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);                                    
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))                     
        .thenReturn(null);                                                                       
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(                      
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    //as instantiateAMProxy is not called at all
    if(!isAMReachableFromClient) {
      return;
    }

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);

    // RM returns AM1 url, null, null and AM2 url on invocations.
    // Nulls simulate the time when AM2 is in the process of restarting.
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn(
        getRunningApplicationReport("am1", 78)).thenReturn(
        getRunningApplicationReport(null, 0)).thenReturn(
        getRunningApplicationReport(null, 0)).thenReturn(
        getRunningApplicationReport("am2", 90));

    GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class);
    when(jobReportResponse1.getJobReport()).thenReturn(
        MRBuilderUtils.newJobReport(jobId, "jobName-firstGen", "user",
            JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null, false));

    // First AM returns a report with jobName firstGen and simulates AM shutdown
    // on second invocation.
    MRClientProtocol firstGenAMProxy = mock(MRClientProtocol.class);
    when(firstGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse1).thenThrow(
            new RuntimeException("AM is down!"));

    GetJobReportResponse jobReportResponse2 = mock(GetJobReportResponse.class);
    when(jobReportResponse2.getJobReport()).thenReturn(
        MRBuilderUtils.newJobReport(jobId, "jobName-secondGen", "user",
            JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null, false));

    // Second AM generation returns a report with jobName secondGen
    MRClientProtocol secondGenAMProxy = mock(MRClientProtocol.class);
    when(secondGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse2);
   
    ClientServiceDelegate clientServiceDelegate = spy(getClientServiceDelegate(
        historyServerProxy, rmDelegate));
    // First time, connection should be to AM1, then to AM2. Further requests
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    //test only applicable when AM not reachable
    if(isAMReachableFromClient) {
      return;
    }

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(                     
        getJobReportResponseFromHistoryServer());                                                

    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn(
        getRunningApplicationReport("am1", 78)).thenReturn(
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 3);

    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED,
        !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(getJobReportResponse());
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(
        new java.lang.reflect.UndeclaredThrowableException(new IOException(
            "Connection refuced1"))).thenThrow(
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

  private void testRMDownForJobStatusBeforeGetAMReport(Configuration conf,
      int noOfRetries) throws YarnRemoteException {
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED,
        !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(
        new java.lang.reflect.UndeclaredThrowableException(new IOException(
            "Connection refuced1"))).thenThrow(
        new java.lang.reflect.UndeclaredThrowableException(new IOException(
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

 
 
  private void testPbServerFactory() {
    InetSocketAddress addr = new InetSocketAddress(0);
    Configuration conf = new Configuration();
    MRClientProtocol instance = new MRClientProtocolTestImpl();
    Server server = null;
    try {
      server =
        RpcServerFactoryPBImpl.get().getServer(
          MRClientProtocol.class, instance, addr, conf, null, 1);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

 
  private void testPbClientFactory() {
    InetSocketAddress addr = new InetSocketAddress(0);
    System.err.println(addr.getHostName() + addr.getPort());
    Configuration conf = new Configuration();
    MRClientProtocol instance = new MRClientProtocolTestImpl();
    Server server = null;
    try {
      server =
        RpcServerFactoryPBImpl.get().getServer(
            MRClientProtocol.class, instance, addr, conf, null, 1);
      server.start();
      System.err.println(server.getListenerAddress());
      System.err.println(NetUtils.getConnectAddress(server));

      MRClientProtocol client = null;
      try {
        client = (MRClientProtocol) RpcClientFactoryPBImpl.get().getClient(MRClientProtocol.class, 1, NetUtils.getConnectAddress(server), conf);
      } catch (YarnException e) {
        e.printStackTrace();
        Assert.fail("Failed to crete client");
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

  @Override
  public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
  throws IOException, InterruptedException {
   
    /* check if we have a hsproxy, if not, no need */
    MRClientProtocol hsProxy = clientCache.getInitializedHSProxy();
    if (hsProxy != null) {
      // JobClient will set this flag if getDelegationToken is called, if so, get
      // the delegation tokens for the HistoryServer also.
      if (conf.getBoolean(JobClient.HS_DELEGATION_TOKEN_REQUIRED,
          DEFAULT_HS_DELEGATION_TOKEN_REQUIRED)) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.