Examples of MRClientProtocol

org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    //as instantiateAMProxy is not called at all
    if(!isAMReachableFromClient) {
      return;
    }


    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);


    // RM returns AM1 url, null, null and AM2 url on invocations.
    // Nulls simulate the time when AM2 is in the process of restarting.
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn(
          getRunningApplicationReport("am1", 78)).thenReturn(
          getRunningApplicationReport(null, 0)).thenReturn(
          getRunningApplicationReport(null, 0)).thenReturn(
          getRunningApplicationReport("am2", 90));
    } catch (YarnException e) {
      throw new IOException(e);
    }


    GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class);
    when(jobReportResponse1.getJobReport()).thenReturn(
        MRBuilderUtils.newJobReport(jobId, "jobName-firstGen", "user",
            JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null,
            false, ""));


    // First AM returns a report with jobName firstGen and simulates AM shutdown
    // on second invocation.
    MRClientProtocol firstGenAMProxy = mock(MRClientProtocol.class);
    when(firstGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse1).thenThrow(
            new RuntimeException("AM is down!"));


    GetJobReportResponse jobReportResponse2 = mock(GetJobReportResponse.class);
    when(jobReportResponse2.getJobReport()).thenReturn(
        MRBuilderUtils.newJobReport(jobId, "jobName-secondGen", "user",
            JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null,
            false, ""));


    // Second AM generation returns a report with jobName secondGen
    MRClientProtocol secondGenAMProxy = mock(MRClientProtocol.class);
    when(secondGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse2);
    
    ClientServiceDelegate clientServiceDelegate = spy(getClientServiceDelegate(
        historyServerProxy, rmDelegate));
    // First time, connection should be to AM1, then to AM2. Further requests

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    //test only applicable when AM not reachable
    if(isAMReachableFromClient) {
      return;
    }


    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(                      
        getJobReportResponseFromHistoryServer());                                                 


    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn(

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 3);


    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED,
        !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(getJobReportResponse());
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(
          new java.lang.reflect.UndeclaredThrowableException(new IOException(

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

  private void testRMDownForJobStatusBeforeGetAMReport(Configuration conf,
      int noOfRetries) throws IOException {
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED,
        !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(
          new java.lang.reflect.UndeclaredThrowableException(new IOException(
              "Connection refuced1"))).thenThrow(

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    
    Task task = job.getTasks().values().iterator().next();
    TaskAttempt attempt = task.getAttempts().values().iterator().next();


    HistoryClientService historyService = historyServer.getClientService();
    MRClientProtocol protocol = historyService.getClientHandler();


    GetTaskAttemptReportRequest gtarRequest = recordFactory
            .newRecordInstance(GetTaskAttemptReportRequest.class);
    // test getTaskAttemptReport
    TaskAttemptId taId = attempt.getID();
    taId.setTaskId(task.getID());
    taId.getTaskId().setJobId(job.getID());
    gtarRequest.setTaskAttemptId(taId);
    GetTaskAttemptReportResponse response = protocol
            .getTaskAttemptReport(gtarRequest);
    assertEquals("container_0_0000_01_000000", response.getTaskAttemptReport()
            .getContainerId().toString());
    assertTrue(response.getTaskAttemptReport().getDiagnosticInfo().isEmpty());
    // counters
    assertNotNull(response.getTaskAttemptReport().getCounters()
            .getCounter(TaskCounter.PHYSICAL_MEMORY_BYTES));
    assertEquals(taId.toString(), response.getTaskAttemptReport()
            .getTaskAttemptId().toString());
    // test getTaskReport
    GetTaskReportRequest request = recordFactory
            .newRecordInstance(GetTaskReportRequest.class);
    TaskId taskId = task.getID();
    taskId.setJobId(job.getID());
    request.setTaskId(taskId);
    GetTaskReportResponse reportResponse = protocol.getTaskReport(request);
    assertEquals("", reportResponse.getTaskReport().getDiagnosticsList()
            .iterator().next());
    // progress
    assertEquals(1.0f, reportResponse.getTaskReport().getProgress(), 0.01);
    // report has corrected taskId
    assertEquals(taskId.toString(), reportResponse.getTaskReport().getTaskId()
            .toString());
    // Task state should be SUCCEEDED
    assertEquals(TaskState.SUCCEEDED, reportResponse.getTaskReport()
            .getTaskState());
    // test getTaskAttemptCompletionEvents
    GetTaskAttemptCompletionEventsRequest taskAttemptRequest = recordFactory
            .newRecordInstance(GetTaskAttemptCompletionEventsRequest.class);
    taskAttemptRequest.setJobId(job.getID());
    GetTaskAttemptCompletionEventsResponse taskAttemptCompletionEventsResponse = protocol
            .getTaskAttemptCompletionEvents(taskAttemptRequest);
    assertEquals(0, taskAttemptCompletionEventsResponse.getCompletionEventCount());
    
    // test getDiagnostics
    GetDiagnosticsRequest diagnosticRequest = recordFactory
            .newRecordInstance(GetDiagnosticsRequest.class);
    diagnosticRequest.setTaskAttemptId(taId);
    GetDiagnosticsResponse diagnosticResponse = protocol
            .getDiagnostics(diagnosticRequest);
    // it is strange : why one empty string ?
    assertEquals(1, diagnosticResponse.getDiagnosticsCount());
    assertEquals("", diagnosticResponse.getDiagnostics(0));

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    UserGroupInformation ugi =
        UserGroupInformation.createRemoteUser("TheDarkLord");
    ugi.addToken(ProtoUtils.convertFromProtoFormat(
        token, jobHistoryServer.getClientService().getBindAddress()));
    final YarnRPC rpc = YarnRPC.create(conf);
    MRClientProtocol userUsingDT =
        ugi.doAs(new PrivilegedAction<MRClientProtocol>() {
          @Override
          public MRClientProtocol run() {
            return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
              jobHistoryServer.getClientService().getBindAddress(), conf);
          }
        });
    GetJobReportRequest jobReportRequest =
        Records.newRecord(GetJobReportRequest.class);
    jobReportRequest.setJobId(MRBuilderUtils.newJobId(123456, 1, 1));
    try {
      userUsingDT.getJobReport(jobReportRequest);
    } catch (YarnRemoteException e) {
      Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
    }
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

  
  
  private void testPbServerFactory() {
    InetSocketAddress addr = new InetSocketAddress(0);
    Configuration conf = new Configuration();
    MRClientProtocol instance = new MRClientProtocolTestImpl();
    Server server = null;
    try {
      server = 
        RpcServerFactoryPBImpl.get().getServer(
          MRClientProtocol.class, instance, addr, conf, null, 1);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

  
  private void testPbClientFactory() {
    InetSocketAddress addr = new InetSocketAddress(0);
    System.err.println(addr.getHostName() + addr.getPort());
    Configuration conf = new Configuration();
    MRClientProtocol instance = new MRClientProtocolTestImpl();
    Server server = null;
    try {
      server = 
        RpcServerFactoryPBImpl.get().getServer(
            MRClientProtocol.class, instance, addr, conf, null, 1);
      server.start();
      System.err.println(server.getListenerAddress());
      System.err.println(NetUtils.getConnectAddress(server));


      MRClientProtocol client = null;
      try {
        client = (MRClientProtocol) RpcClientFactoryPBImpl.get().getClient(MRClientProtocol.class, 1, NetUtils.getConnectAddress(server), conf);
      } catch (YarnException e) {
        e.printStackTrace();
        Assert.fail("Failed to crete client");

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol


  MRClientProtocol instantiateAMProxy(final InetSocketAddress serviceAddr)
      throws IOException {
    LOG.trace("Connecting to ApplicationMaster at: " + serviceAddr);
    YarnRPC rpc = YarnRPC.create(conf);
    MRClientProtocol proxy = 
         (MRClientProtocol) rpc.getProxy(MRClientProtocol.class,
            serviceAddr, conf);
    LOG.trace("Connected to ApplicationMaster at: " + serviceAddr);
    return proxy;
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

  @Override
  public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
  throws IOException, InterruptedException {
    
    /* check if we have a hsproxy, if not, no need */
    MRClientProtocol hsProxy = clientCache.getInitializedHSProxy();
    if (hsProxy != null) {
      // JobClient will set this flag if getDelegationToken is called, if so, get
      // the delegation tokens for the HistoryServer also.
      if (conf.getBoolean(JobClient.HS_DELEGATION_TOKEN_REQUIRED, 
          DEFAULT_HS_DELEGATION_TOKEN_REQUIRED)) {

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.