Examples of RMStateStore


Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore

  private static boolean isQueueUser = false;

  @BeforeClass
  public static void setup() throws InterruptedException, IOException {
    RMStateStore store = RMStateStoreFactory.getStore(conf);
    conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
    AccessControlList adminACL = new AccessControlList("");
    adminACL.addGroup(SUPER_GROUP);
    conf.set(YarnConfiguration.YARN_ADMIN_ACL, adminACL.getAclString());
    resourceManager = new MockRM(conf) {
View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore

  @SuppressWarnings ("rawtypes")
  public void testAppSubmit() throws Exception {
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    RMStateStore stateStore = mock(RMStateStore.class);
    when(rmContext.getStateStore()).thenReturn(stateStore);
    RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler,
        null, mock(ApplicationACLsManager.class), new Configuration());
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(
        new EventHandler<Event>() {
View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore

     */
    // Basic setup
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    RMStateStore stateStore = mock(RMStateStore.class);
    when(rmContext.getStateStore()).thenReturn(stateStore);
    RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler,
        null, mock(ApplicationACLsManager.class), new Configuration());
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(
        new EventHandler<Event>() {
View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore

      throws IOException, InterruptedException, BrokenBarrierException,
      YarnException {
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    RMStateStore stateStore = mock(RMStateStore.class);
    when(rmContext.getStateStore()).thenReturn(stateStore);
    RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler,
        null, mock(ApplicationACLsManager.class), new Configuration());

    final ApplicationId appId1 = getApplicationId(100);
View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore

      // Set the masterContainer
      appAttempt.setMasterContainer(amContainerAllocation.getContainers().get(
                                                                           0));
      appAttempt.getSubmissionContext().setResource(
          appAttempt.getMasterContainer().getResource());
      RMStateStore store = appAttempt.rmContext.getStateStore();
      appAttempt.storeAttempt(store);
    }
View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore

          assert (amContainerAllocation.getContainers().size() == 0);
        }
        return RMAppAttemptState.SCHEDULED;
      } else {
        // RM not allocating container. AM is self launched.
        RMStateStore store = appAttempt.rmContext.getStateStore();
        // save state and then go to LAUNCHED state
        appAttempt.storeAttempt(store);
        return RMAppAttemptState.LAUNCHED_UNMANAGED_SAVING;
      }
    }
View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore

    return credentials;
  }
 
  @Override
  public void recover(RMState state) throws Exception {
    RMStateStore store = rmContext.getStateStore();
    assert store != null;
    // recover applications
    Map<ApplicationId, ApplicationState> appStates = state.getApplicationState();
    LOG.info("Recovering " + appStates.size() + " applications");
    for(ApplicationState appState : appStates.values()) {
      boolean shouldRecover = true;
      if(appState.getApplicationSubmissionContext().getUnmanagedAM()) {
        // do not recover unmanaged applications since current recovery
        // mechanism of restarting attempts does not work for them.
        // This will need to be changed in work preserving recovery in which
        // RM will re-connect with the running AM's instead of restarting them
        LOG.info("Not recovering unmanaged application " + appState.getAppId());
        shouldRecover = false;
      }
      int individualMaxAppAttempts = appState.getApplicationSubmissionContext()
          .getMaxAppAttempts();
      int maxAppAttempts;
      if (individualMaxAppAttempts <= 0 ||
          individualMaxAppAttempts > globalMaxAppAttempts) {
        maxAppAttempts = globalMaxAppAttempts;
        LOG.warn("The specific max attempts: " + individualMaxAppAttempts
            + " for application: " + appState.getAppId()
            + " is invalid, because it is out of the range [1, "
            + globalMaxAppAttempts + "]. Use the global max attempts instead.");
      } else {
        maxAppAttempts = individualMaxAppAttempts;
      }
      // In work-preserve restart, if attemptCount == maxAttempts, the job still
      // needs to be recovered because the last attempt may still be running.
      if(appState.getAttemptCount() >= maxAppAttempts) {
        LOG.info("Not recovering application " + appState.getAppId() +
            " due to recovering attempt is beyond maxAppAttempt limit");
        shouldRecover = false;
      }

      // re-submit the application
      // this is going to send an app start event but since the async dispatcher
      // has not started that event will be queued until we have completed re
      // populating the state
      if(shouldRecover) {
        LOG.info("Recovering application " + appState.getAppId());
        submitApplication(appState.getApplicationSubmissionContext(),
                        appState.getSubmitTime(), true, appState.getUser());
        // re-populate attempt information in application
        RMAppImpl appImpl = (RMAppImpl) rmContext.getRMApps().get(
                                                        appState.getAppId());
        appImpl.recover(state);
      }
      else {
        store.removeApplication(appState);
      }
    }
  }
View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore

   
    boolean isRecoveryEnabled = conf.getBoolean(
        YarnConfiguration.RECOVERY_ENABLED,
        YarnConfiguration.DEFAULT_RM_RECOVERY_ENABLED);
   
    RMStateStore rmStore = null;
    if(isRecoveryEnabled) {
      recoveryEnabled = true;
      rmStore =  RMStateStoreFactory.getStore(conf);
    } else {
      recoveryEnabled = false;
      rmStore = new NullRMStateStore();
    }

    try {
      rmStore.init(conf);
      rmStore.setRMDispatcher(rmDispatcher);
    } catch (Exception e) {
      // the Exception from stateStore.init() needs to be handled for
      // HA and we need to give up master status if we got fenced
      LOG.error("Failed to init state store", e);
      ExitUtil.terminate(1, e);
View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore

    this.amRmTokenSecretManager.start();
    this.containerTokenSecretManager.start();
    this.nmTokenSecretManager.start();

    RMStateStore rmStore = rmContext.getStateStore();
    // The state store needs to start irrespective of recoveryEnabled as apps
    // need events to move to further states.
    rmStore.start();

    if(recoveryEnabled) {
      try {
        RMState state = rmStore.loadState();
        recover(state);
      } catch (Exception e) {
        // the Exception from loadState() needs to be handled for
        // HA and we need to give up master status if we got fenced
        LOG.error("Failed to load/recover state", e);
View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore

    }*/

    DefaultMetricsSystem.shutdown();

    if (rmContext != null) {
      RMStateStore store = rmContext.getStateStore();
      try {
        store.close();
      } catch (Exception e) {
        LOG.error("Error closing store.", e);
      }
    }

View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.