Examples of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp

Package org.apache.hadoop.yarn.server.resourcemanager.scheduler

Examples of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp

org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp
Represents an Application from the viewpoint of the scheduler. Each running Application in the RM corresponds to one instance of this class.

  }
  
  private synchronized void addApplication(ApplicationAttemptId appAttemptId,
      String user) {
    // TODO: Fix store
    SchedulerApp schedulerApp = 
        new SchedulerApp(appAttemptId, user, DEFAULT_QUEUE, activeUsersManager,
            this.rmContext, null);
    applications.put(appAttemptId, schedulerApp);
    metrics.submitApp(user, appAttemptId.getAttemptId());
    LOG.info("Application Submission: " + appAttemptId.getApplicationId() + 
        " from " + user + ", currently active: " + applications.size());

View Full Code Here


  private synchronized void doneApplication(
      ApplicationAttemptId applicationAttemptId,
      RMAppAttemptState rmAppAttemptFinalState)
      throws IOException {
    SchedulerApp application = getApplication(applicationAttemptId);
    if (application == null) {
      throw new IOException("Unknown application " + applicationAttemptId + 
      " has completed!");
    }


    // Kill all 'live' containers
    for (RMContainer container : application.getLiveContainers()) {
      containerCompleted(container, 
          SchedulerUtils.createAbnormalContainerStatus(
              container.getContainerId(), 
              SchedulerUtils.COMPLETED_APPLICATION),
          RMContainerEventType.KILL);
    }


    // Inform the activeUsersManager
    synchronized (application) {
      activeUsersManager.deactivateApplication(
          application.getUser(), application.getApplicationId());
    }


    // Clean up pending requests, metrics etc.
    application.stop(rmAppAttemptFinalState);


    // Remove the application
    applications.remove(applicationAttemptId);
  }

View Full Code Here

        " #applications=" + applications.size());


    // Try to assign containers to applications in fifo order
    for (Map.Entry<ApplicationAttemptId, SchedulerApp> e : applications
        .entrySet()) {
      SchedulerApp application = e.getValue();
      LOG.debug("pre-assignContainers");
      application.showRequests();
      synchronized (application) {
        for (Priority priority : application.getPriorities()) {
          int maxContainers = 
            getMaxAllocatableContainers(application, priority, node, 
                NodeType.OFF_SWITCH); 
          // Ensure the application needs containers of this priority
          if (maxContainers > 0) {
            int assignedContainers = 
              assignContainersOnNode(node, application, priority);
            // Do not assign out of order w.r.t priorities
            if (assignedContainers == 0) {
              break;
            }
          }
        }
      }
      
      LOG.debug("post-assignContainers");
      application.showRequests();


      // Done
      if (Resources.lessThan(node.getAvailableResource(), minimumAllocation)) {
        break;
      }
    }


    // Update the applications' headroom to correctly take into
    // account the containers assigned in this update.
    for (SchedulerApp application : applications.values()) {
      application.setHeadroom(Resources.subtract(clusterResource, usedResource));
    }
  }

View Full Code Here

  }


  private void containerLaunchedOnNode(ContainerId containerId, SchedulerNode node) {
    // Get the application for the finished container
    ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId();
    SchedulerApp application = getApplication(applicationAttemptId);
    if (application == null) {
      LOG.info("Unknown application: " + applicationAttemptId + 
          " launched container " + containerId +
          " on node: " + node);
      // Some unknown container sneaked into the system. Kill it.
      this.rmContext.getDispatcher().getEventHandler()
        .handle(new RMNodeCleanContainerEvent(node.getNodeID(), containerId));


      return;
    }
    
    application.containerLaunchedOnNode(containerId, node.getNodeID());
  }

View Full Code Here

    }


    // Get the application for the finished container
    Container container = rmContainer.getContainer();
    ApplicationAttemptId applicationAttemptId = container.getId().getApplicationAttemptId();
    SchedulerApp application = getApplication(applicationAttemptId);
    
    // Get the node on which the container was allocated
    SchedulerNode node = getNode(container.getNodeId());
    
    if (application == null) {
      LOG.info("Unknown application: " + applicationAttemptId + 
          " released container " + container.getId() +
          " on node: " + node + 
          " with event: " + event);
      return;
    }


    // Inform the application
    application.containerCompleted(rmContainer, containerStatus, event);


    // Inform the node
    node.releaseContainer(container);
    
    // Update total usage

View Full Code Here

    SchedulerNode node = getNode(nodeId);
    return node == null ? null : new SchedulerNodeReport(node);
  }
  
  private RMContainer getRMContainer(ContainerId containerId) {
    SchedulerApp application = 
        getApplication(containerId.getApplicationAttemptId());
    return (application == null) ? null : application.getRMContainer(containerId);
  }

View Full Code Here

    doAnswer(
        new Answer<Container>() {
          @Override
          public Container answer(InvocationOnMock invocation) 
              throws Throwable {
            final SchedulerApp application = 
                (SchedulerApp)(invocation.getArguments()[0]);
            final ContainerId containerId =                 
                TestUtils.getMockContainerId(application);


            Container container = TestUtils.getMockContainer(

View Full Code Here

    final String user_0 = "user_0";


    // Submit applications
    final ApplicationAttemptId appAttemptId_0 = 
        TestUtils.getMockApplicationAttemptId(0, 0); 
    SchedulerApp app_0 = 
        new SchedulerApp(appAttemptId_0, user_0, a, 
            mock(ActiveUsersManager.class), rmContext, null);
    a.submitApplication(app_0, user_0, B);


    final ApplicationAttemptId appAttemptId_1 = 
        TestUtils.getMockApplicationAttemptId(1, 0); 
    SchedulerApp app_1 = 
        new SchedulerApp(appAttemptId_1, user_0, a, 
            mock(ActiveUsersManager.class), rmContext, null);
    a.submitApplication(app_1, user_0, B);  // same user


    
    // Setup some nodes

View Full Code Here

    final String user_d = "user_d";


    // Submit applications
    final ApplicationAttemptId appAttemptId_0 = TestUtils
        .getMockApplicationAttemptId(0, 1);
    SchedulerApp app_0 = new SchedulerApp(appAttemptId_0, user_d, d, null,
        rmContext, null);
    d.submitApplication(app_0, user_d, D);


    // Attempt the same application again
    final ApplicationAttemptId appAttemptId_1 = TestUtils
        .getMockApplicationAttemptId(0, 2);
    SchedulerApp app_1 = new SchedulerApp(appAttemptId_1, user_d, d, null,
        rmContext, null);
    d.submitApplication(app_1, user_d, D); // same user
  }

View Full Code Here

    final String user_0 = "user_0";


    // Submit applications
    final ApplicationAttemptId appAttemptId_0 = TestUtils
        .getMockApplicationAttemptId(0, 1);
    SchedulerApp app_0 = new SchedulerApp(appAttemptId_0, user_0, a, null,
        rmContext, null);
    a.submitApplication(app_0, user_0, B);
    
    when(cs.getApplication(appAttemptId_0)).thenReturn(app_0);
    AppRemovedSchedulerEvent event = new AppRemovedSchedulerEvent(
        appAttemptId_0, RMAppAttemptState.FAILED);
    cs.handle(event);
    
    assertEquals(0, a.getMetrics().getAppsPending());
    assertEquals(1, a.getMetrics().getAppsFailed());


    // Attempt the same application again
    final ApplicationAttemptId appAttemptId_1 = TestUtils
        .getMockApplicationAttemptId(0, 2);
    SchedulerApp app_1 = new SchedulerApp(appAttemptId_1, user_0, a, null,
        rmContext, null);
    a.submitApplication(app_1, user_0, B); // same user


    assertEquals(1, a.getMetrics().getAppsSubmitted());
    assertEquals(1, a.getMetrics().getAppsPending());

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp

org.apache.hadoop.yarn.api.records.Container

org.apache.hadoop.yarn.api.records.ContainerId

org.apache.hadoop.yarn.api.records.Resource

org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer

org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent

org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent

org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl

org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerReservedEvent

org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent

org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.