Examples of MRClientProtocol


Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    final long initialInterval = 10000l;
    final long maxLifetime= 20000l;
    final long renewInterval = 10000l;

    JobHistoryServer jobHistoryServer = null;
    MRClientProtocol clientUsingDT = null;
    long tokenFetchTime;
    try {
      jobHistoryServer = new JobHistoryServer() {
        protected void doSecureLogin(Configuration conf) throws IOException {
          // no keytab based login
        };

        @Override
        protected JHSDelegationTokenSecretManager createJHSSecretManager(
            Configuration conf, HistoryServerStateStoreService store) {
          return new JHSDelegationTokenSecretManager(initialInterval,
              maxLifetime, renewInterval, 3600000, store);
        }

        @Override
        protected HistoryClientService createHistoryClientService() {
          return new HistoryClientService(historyContext,
            this.jhsDTSecretManager) {
            @Override
            protected void initializeWebApp(Configuration conf) {
              // Don't need it, skip.;
              }
          };
        }
      };
//      final JobHistoryServer jobHistoryServer = jhServer;
      jobHistoryServer.init(conf);
      jobHistoryServer.start();
      final MRClientProtocol hsService = jobHistoryServer.getClientService()
          .getClientHandler();

      // Fake the authentication-method
      UserGroupInformation loggedInUser = UserGroupInformation
          .createRemoteUser("testrenewer@APACHE.ORG");
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

      final InetSocketAddress hsAddress, String user, final Configuration conf) {
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
    ugi.addToken(ConverterUtils.convertFromYarn(token, hsAddress));

    final YarnRPC rpc = YarnRPC.create(conf);
    MRClientProtocol hsWithDT = ugi
        .doAs(new PrivilegedAction<MRClientProtocol>() {

          @Override
          public MRClientProtocol run() {
            return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    return Arrays.asList(data);
  }

  @Test
  public void testUnknownAppInRM() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(
        getJobReportResponse());
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(
        historyServerProxy, getRMDelegate());

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

  }

  @Test
  public void testRemoteExceptionFromHistoryServer() throws Exception {

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(
        new IOException("Job ID doesnot Exist"));

    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
        .thenReturn(null);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

  }

  @Test
  public void testRetriesOnConnectionFailure() throws Exception {

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(
        new RuntimeException("1")).thenThrow(new RuntimeException("2"))      
        .thenReturn(getJobReportResponse());

    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
      .thenReturn(getRunningApplicationReport("am1", 78));

    // throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and
    // succeed in the 5th call.
    final MRClientProtocol amProxy = mock(MRClientProtocol.class);
    when(amProxy.getJobReport(any(GetJobReportRequest.class)))
      .thenThrow(new RuntimeException("11"))
      .thenThrow(new RuntimeException("22"))
      .thenThrow(new RuntimeException("33"))
      .thenThrow(new RuntimeException("44")).thenReturn(getJobReportResponse());
    Configuration conf = new YarnConfiguration();
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState());
  }
 
  @Test
  public void testJobReportFromHistoryServer() throws Exception {                                
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);                          
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(                     
        getJobReportResponseFromHistoryServer());                                                
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);                                    
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))                     
    .thenReturn(null);                                                                       
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(                      
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    Assert.assertEquals(1.0f, jobStatus.getReduceProgress(), 0.0f);
  }
 
  @Test
  public void testCountersFromHistoryServer() throws Exception {                                
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);                          
    when(historyServerProxy.getCounters(getCountersRequest())).thenReturn(                     
        getCountersResponseFromHistoryServer());
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);                                    
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))                     
        .thenReturn(null);                                                                       
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(                      
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    //as instantiateAMProxy is not called at all
    if(!isAMReachableFromClient) {
      return;
    }

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);

    // RM returns AM1 url, null, null and AM2 url on invocations.
    // Nulls simulate the time when AM2 is in the process of restarting.
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn(
          getRunningApplicationReport("am1", 78)).thenReturn(
          getRunningApplicationReport(null, 0)).thenReturn(
          getRunningApplicationReport(null, 0)).thenReturn(
          getRunningApplicationReport("am2", 90));
    } catch (YarnException e) {
      throw new IOException(e);
    }

    GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class);
    when(jobReportResponse1.getJobReport()).thenReturn(
        MRBuilderUtils.newJobReport(jobId, "jobName-firstGen", "user",
            JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null,
            false, ""));

    // First AM returns a report with jobName firstGen and simulates AM shutdown
    // on second invocation.
    MRClientProtocol firstGenAMProxy = mock(MRClientProtocol.class);
    when(firstGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse1).thenThrow(
            new RuntimeException("AM is down!"));

    GetJobReportResponse jobReportResponse2 = mock(GetJobReportResponse.class);
    when(jobReportResponse2.getJobReport()).thenReturn(
        MRBuilderUtils.newJobReport(jobId, "jobName-secondGen", "user",
            JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null,
            false, ""));

    // Second AM generation returns a report with jobName secondGen
    MRClientProtocol secondGenAMProxy = mock(MRClientProtocol.class);
    when(secondGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse2);
   
    ClientServiceDelegate clientServiceDelegate = spy(getClientServiceDelegate(
        historyServerProxy, rmDelegate));
    // First time, connection should be to AM1, then to AM2. Further requests
View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    //test only applicable when AM not reachable
    if(isAMReachableFromClient) {
      return;
    }

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(                     
        getJobReportResponseFromHistoryServer());                                                

    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn(
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.