Examples of DrainDispatcher


Examples of org.apache.hadoop.yarn.event.DrainDispatcher

    final Application app;

    WrappedApplication(int id, long timestamp, String user, int numContainers) {
      Configuration conf = new Configuration();
     
      dispatcher = new DrainDispatcher();
      containerTokenIdentifierMap =
          new HashMap<ContainerId, ContainerTokenIdentifier>();
      dispatcher.init(conf);

      localizerBus = mock(EventHandler.class);
View Full Code Here

Examples of org.apache.hadoop.yarn.event.DrainDispatcher

  public void testFailAbortDoesntHang() throws IOException {
    Configuration conf = new Configuration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    conf.set(MRJobConfig.MR_AM_COMMITTER_CANCEL_TIMEOUT_MS, "1000");
   
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    OutputCommitter committer = Mockito.mock(OutputCommitter.class);
    CommitterEventHandler commitHandler =
        createCommitterEventHandler(dispatcher, committer);
    commitHandler.init(conf);
    commitHandler.start();
    //Job has only 1 mapper task. No reducers
    conf.setInt(MRJobConfig.NUM_REDUCES, 0);
    conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 1);
    JobImpl job = createRunningStubbedJob(conf, dispatcher, 1, null);

    //Fail / finish all the tasks. This should land the JobImpl directly in the
    //FAIL_ABORT state
    for(Task t: job.tasks.values()) {
      TaskImpl task = (TaskImpl) t;
      task.handle(new TaskEvent(task.getID(), TaskEventType.T_SCHEDULE));
      for(TaskAttempt ta: task.getAttempts().values()) {
        task.handle(new TaskTAttemptEvent(ta.getID(),
          TaskEventType.T_ATTEMPT_FAILED));
      }
    }
    assertJobState(job, JobStateInternal.FAIL_ABORT);

    dispatcher.await();
    //Verify abortJob is called once and the job failed
    Mockito.verify(committer, Mockito.timeout(2000).times(1))
      .abortJob((JobContext) Mockito.any(), (State) Mockito.any());
    assertJobState(job, JobStateInternal.FAILED);

    dispatcher.stop();
  }
View Full Code Here

Examples of org.apache.hadoop.yarn.event.DrainDispatcher

      authMethod = AuthenticationMethod.KERBEROS;
    }
    SecurityUtil.setAuthenticationMethod(authMethod, conf);
    UserGroupInformation.setConfiguration(conf);

    rmDispatcher = new DrainDispatcher();
    ContainerAllocationExpirer containerAllocationExpirer =
        mock(ContainerAllocationExpirer.class);
    AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class);
    AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class);
    store = mock(RMStateStore.class);
View Full Code Here

Examples of org.apache.hadoop.yarn.event.DrainDispatcher

    ContainerManagementProtocol containerManager =
        mock(ContainerManagementProtocol.class);
    StartContainersResponse mockResponse = mock(StartContainersResponse.class);
    when(containerManager.startContainers((StartContainersRequest) any()))
      .thenReturn(mockResponse);
    final DrainDispatcher dispatcher = new DrainDispatcher();

    MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {
      protected ClientRMService createClientRMService() {
        return new ClientRMService(this.rmContext, scheduler,
          this.rmAppManager, this.applicationACLsManager,
          this.rmDTSecretManager);
      };

      @Override
      protected Dispatcher createDispatcher() {
        return dispatcher;
      }

      @Override
      protected void doSecureLogin() throws IOException {
      }
    };
    rm.start();

    // Submit an app
    RMApp app = rm.submitApp(1024);

    // Set up a node.
    MockNM nm1 = rm.registerNode("localhost:1234", 3072);
    nm1.nodeHeartbeat(true);
    dispatcher.await();
   

    nm1.nodeHeartbeat(true);
    dispatcher.await();

    // Get the app-report.
    GetApplicationReportRequest request =
        Records.newRecord(GetApplicationReportRequest.class);
    request.setApplicationId(app.getApplicationId());
View Full Code Here

Examples of org.apache.hadoop.yarn.event.DrainDispatcher

      MyResourceManager.setClusterTimeStamp(fakeClusterTimeStamp);
    }

    @Override
    protected Dispatcher createDispatcher() {
      return new DrainDispatcher();
    }
View Full Code Here

Examples of org.apache.hadoop.yarn.event.DrainDispatcher

  }

  private void validateRMNMKeyExchange(YarnConfiguration conf) throws Exception {
    // Default rolling and activation intervals are large enough, no need to
    // intervene
    final DrainDispatcher dispatcher = new DrainDispatcher();
    ResourceManager rm = new ResourceManager() {

      @Override
      protected void doSecureLogin() throws IOException {
        // Do nothing.
      }

      @Override
      protected Dispatcher createDispatcher() {
        return dispatcher;
      }
      @Override
      protected void startWepApp() {
        // Don't need it, skip.
      }
    };
    rm.init(conf);
    rm.start();

    // Testing ContainerToken and NMToken
    String containerToken = "Container Token : ";
    String nmToken = "NM Token : ";
   
    MockNM nm = new MockNM("host:1234", 3072, rm.getResourceTrackerService());
    RegisterNodeManagerResponse registrationResponse = nm.registerNode();
   
    MasterKey containerTokenMasterKey =
        registrationResponse.getContainerTokenMasterKey();
    Assert.assertNotNull(containerToken
        + "Registration should cause a key-update!", containerTokenMasterKey);
    MasterKey nmTokenMasterKey = registrationResponse.getNMTokenMasterKey();
    Assert.assertNotNull(nmToken
        + "Registration should cause a key-update!", nmTokenMasterKey);
   
    dispatcher.await();

    NodeHeartbeatResponse response = nm.nodeHeartbeat(true);
    Assert.assertNull(containerToken +
        "First heartbeat after registration shouldn't get any key updates!",
        response.getContainerTokenMasterKey());
    Assert.assertNull(nmToken +
        "First heartbeat after registration shouldn't get any key updates!",
        response.getNMTokenMasterKey());
    dispatcher.await();

    response = nm.nodeHeartbeat(true);
    Assert.assertNull(containerToken +
        "Even second heartbeat after registration shouldn't get any key updates!",
        response.getContainerTokenMasterKey());
    Assert.assertNull(nmToken +
        "Even second heartbeat after registration shouldn't get any key updates!",
        response.getContainerTokenMasterKey());
   
    dispatcher.await();

    // Let's force a roll-over
    rm.getRMContext().getContainerTokenSecretManager().rollMasterKey();
    rm.getRMContext().getNMTokenSecretManager().rollMasterKey();

    // Heartbeats after roll-over and before activation should be fine.
    response = nm.nodeHeartbeat(true);
    Assert.assertNotNull(containerToken +
        "Heartbeats after roll-over and before activation should not err out.",
        response.getContainerTokenMasterKey());
    Assert.assertNotNull(nmToken +
        "Heartbeats after roll-over and before activation should not err out.",
        response.getNMTokenMasterKey());
   
    Assert.assertEquals(containerToken +
        "Roll-over should have incremented the key-id only by one!",
        containerTokenMasterKey.getKeyId() + 1,
        response.getContainerTokenMasterKey().getKeyId());
    Assert.assertEquals(nmToken +
        "Roll-over should have incremented the key-id only by one!",
        nmTokenMasterKey.getKeyId() + 1,
        response.getNMTokenMasterKey().getKeyId());
    dispatcher.await();

    response = nm.nodeHeartbeat(true);
    Assert.assertNull(containerToken +
        "Second heartbeat after roll-over shouldn't get any key updates!",
        response.getContainerTokenMasterKey());
    Assert.assertNull(nmToken +
        "Second heartbeat after roll-over shouldn't get any key updates!",
        response.getNMTokenMasterKey());
    dispatcher.await();

    // Let's force activation
    rm.getRMContext().getContainerTokenSecretManager().activateNextMasterKey();
    rm.getRMContext().getNMTokenSecretManager().activateNextMasterKey();

    response = nm.nodeHeartbeat(true);
    Assert.assertNull(containerToken
        + "Activation shouldn't cause any key updates!",
        response.getContainerTokenMasterKey());
    Assert.assertNull(nmToken
        + "Activation shouldn't cause any key updates!",
        response.getNMTokenMasterKey());
    dispatcher.await();

    response = nm.nodeHeartbeat(true);
    Assert.assertNull(containerToken +
        "Even second heartbeat after activation shouldn't get any key updates!",
        response.getContainerTokenMasterKey());
    Assert.assertNull(nmToken +
        "Even second heartbeat after activation shouldn't get any key updates!",
        response.getNMTokenMasterKey());
    dispatcher.await();

    rm.stop();
  }
View Full Code Here

Examples of org.apache.hadoop.yarn.event.DrainDispatcher

   
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
        this.remoteRootLogDir.getAbsolutePath());
   
    DrainDispatcher dispatcher = createDispatcher();
    EventHandler<ApplicationEvent> appEventHandler = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, appEventHandler);
   
    LogAggregationService logAggregationService = spy(
        new LogAggregationService(dispatcher, this.context, this.delSrvc,
                                  super.dirsHandler));
    logAggregationService.init(this.conf);
    logAggregationService.start();
   
    ApplicationId appId = BuilderUtils.newApplicationId(
        System.currentTimeMillis(), (int)Math.random());
    Exception e = new RuntimeException("KABOOM!");
    doThrow(e)
      .when(logAggregationService).createAppDir(any(String.class),
          any(ApplicationId.class), any(UserGroupInformation.class));
    logAggregationService.handle(new LogHandlerAppStartedEvent(appId,
        this.user, null,
        ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY, this.acls));       
   
    dispatcher.await();
    ApplicationEvent expectedEvents[] = new ApplicationEvent[]{
        new ApplicationEvent(appId,
            ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED)
    };
    checkEvents(appEventHandler, expectedEvents, false,
        "getType", "getApplicationID", "getDiagnostic");
    // filesystems may have been instantiated
    verify(logAggregationService).closeFileSystems(
        any(UserGroupInformation.class));

    // verify trying to collect logs for containers/apps we don't know about
    // doesn't blow up and tear down the NM
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(
        BuilderUtils.newContainerId(4, 1, 1, 1), 0));
    dispatcher.await();
    logAggregationService.handle(new LogHandlerAppFinishedEvent(
        BuilderUtils.newApplicationId(1, 5)));
    dispatcher.await();

    logAggregationService.stop();
    assertEquals(0, logAggregationService.getNumAggregators());
  }
View Full Code Here

Examples of org.apache.hadoop.yarn.event.DrainDispatcher

          logAcls.get(appAccessType));
    }
  }

  private DrainDispatcher createDispatcher() {
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(this.conf);
    dispatcher.start();
    return dispatcher;
  }
View Full Code Here

Examples of org.apache.hadoop.yarn.event.DrainDispatcher

    // get the AppLogAggregationImpl thread to crash
    LocalDirsHandlerService mockedDirSvc = mock(LocalDirsHandlerService.class);
    when(mockedDirSvc.getLogDirs()).thenThrow(new RuntimeException());

    DrainDispatcher dispatcher = createDispatcher();
    EventHandler<ApplicationEvent> appEventHandler = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, appEventHandler);

    LogAggregationService logAggregationService =
        new LogAggregationService(dispatcher, this.context, delSrvc,
                                  mockedDirSvc);
    logAggregationService.init(this.conf);
View Full Code Here

Examples of org.apache.hadoop.yarn.event.DrainDispatcher

    DeletionService delSrvc = mock(DeletionService.class);

    // get the AppLogAggregationImpl thread to crash
    LocalDirsHandlerService mockedDirSvc = mock(LocalDirsHandlerService.class);

    DrainDispatcher dispatcher = createDispatcher();
    EventHandler<ApplicationEvent> appEventHandler = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, appEventHandler);

    LogAggregationService logAggregationService =
        new LogAggregationService(dispatcher, this.context, delSrvc,
                                  mockedDirSvc);
    logAggregationService.init(this.conf);
    logAggregationService.start();

    ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
    logAggregationService.handle(new LogHandlerAppStartedEvent(
            application1, this.user, null,
            ContainerLogsRetentionPolicy.ALL_CONTAINERS, this.acls));

    logAggregationService.handle(new LogHandlerAppFinishedEvent(application1));
    dispatcher.await();
    int timeToWait = 20 * 1000;
    while (timeToWait > 0 && logAggregationService.getNumAggregators() > 0) {
      Thread.sleep(100);
      timeToWait -= 100;
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.