Examples of AllocateResponse

org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse
The response sent by the ResourceManager the ApplicationMaster during resource negotiation.

The response, includes:
- Response ID to track duplicate responses.
- An AMCommand sent by ResourceManager to let the ApplicationMaster take some actions (resync, shutdown etc.).
- A list of newly allocated {@link Container}.
- A list of completed {@link Container}s' statuses.
- The available headroom for resources in the cluster for the application.
- A list of nodes whose status has been updated.
- The number of available nodes in a cluster.
- A description of resources requested back by the cluster
@see ApplicationMasterProtocol#allocate(AllocateRequest)

Examples of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

    RMAppAttempt attempt = app.getCurrentAppAttempt();
    MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());


    // request for containers
    int request = 2;
    AllocateResponse ar =
        am.allocate("h1", 1000, request, new ArrayList<ContainerId>());
    Assert.assertTrue(ar.getAMCommand() == AMCommand.AM_RESYNC);


    // kick the scheduler
    nm1.nodeHeartbeat(true);
    AllocateResponse amrs =
        am.allocate(new ArrayList<ResourceRequest>(),
          new ArrayList<ContainerId>());
    Assert.assertTrue(ar.getAMCommand() == AMCommand.AM_RESYNC);


    am.registerAppAttempt();
    thrown = false;
    try {
    am.registerAppAttempt(false);
    }
    catch (Exception e) {
      Assert.assertEquals("Application Master is already registered : "
          + attempt.getAppAttemptId().getApplicationId(),
        e.getMessage());
      thrown = true;
    }
    Assert.assertTrue(thrown);


    // Simulate an AM that was disconnected and app attempt was removed
    // (responseMap does not contain attemptid)
    am.unregisterAppAttempt();
    nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1,
        ContainerState.COMPLETE);
    am.waitForState(RMAppAttemptState.FINISHED);


    AllocateResponse amrs2 =
        am.allocate(new ArrayList<ResourceRequest>(),
            new ArrayList<ContainerId>());
    Assert.assertTrue(amrs2.getAMCommand() == AMCommand.AM_SHUTDOWN);
  }

View Full Code Here

Examples of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

        nm2.getNodeId());
    Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemory());


    // add request for containers
    am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, GB, 1, 1);
    AllocateResponse alloc1Response = am1.schedule(); // send the request
    // add request for containers
    am2.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 0, 1);
    AllocateResponse alloc2Response = am2.schedule(); // send the request


    // kick the scheduler, 1 GB and 3 GB given to AM1 and AM2, remaining 0
    nm1.nodeHeartbeat(true);
    while (alloc1Response.getAllocatedContainers().size() < 1) {
      LOG.info("Waiting for containers to be created for app 1...");
      Thread.sleep(1000);
      alloc1Response = am1.schedule();
    }
    while (alloc2Response.getAllocatedContainers().size() < 1) {
      LOG.info("Waiting for containers to be created for app 2...");
      Thread.sleep(1000);
      alloc2Response = am2.schedule();
    }
    // kick the scheduler, nothing given remaining 2 GB.
    nm2.nodeHeartbeat(true);


    List<Container> allocated1 = alloc1Response.getAllocatedContainers();
    Assert.assertEquals(1, allocated1.size());
    Assert.assertEquals(1 * GB, allocated1.get(0).getResource().getMemory());
    Assert.assertEquals(nm1.getNodeId(), allocated1.get(0).getNodeId());


    List<Container> allocated2 = alloc2Response.getAllocatedContainers();
    Assert.assertEquals(1, allocated2.size());
    Assert.assertEquals(3 * GB, allocated2.get(0).getResource().getMemory());
    Assert.assertEquals(nm1.getNodeId(), allocated2.get(0).getNodeId());
    
    report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());

View Full Code Here

Examples of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

    am1.registerAppAttempt();


    // allocate request returns no updated node
    AllocateRequest allocateRequest1 =
        AllocateRequest.newInstance(0, 0F, null, null, null);
    AllocateResponse response1 =
        allocate(attempt1.getAppAttemptId(), allocateRequest1);
    List<NodeReport> updatedNodes = response1.getUpdatedNodes();
    Assert.assertEquals(0, updatedNodes.size());


    syncNodeHeartbeat(nm4, false);
    
    // allocate request returns updated node
    allocateRequest1 =
        AllocateRequest.newInstance(response1.getResponseId(), 0F, null, null,
          null);
    response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1);
    updatedNodes = response1.getUpdatedNodes();
    Assert.assertEquals(1, updatedNodes.size());
    NodeReport nr = updatedNodes.iterator().next();
    Assert.assertEquals(nm4.getNodeId(), nr.getNodeId());
    Assert.assertEquals(NodeState.UNHEALTHY, nr.getNodeState());
    
    // resending the allocate request returns the same result
    response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1);
    updatedNodes = response1.getUpdatedNodes();
    Assert.assertEquals(1, updatedNodes.size());
    nr = updatedNodes.iterator().next();
    Assert.assertEquals(nm4.getNodeId(), nr.getNodeId());
    Assert.assertEquals(NodeState.UNHEALTHY, nr.getNodeState());


    syncNodeLost(nm3);
    
    // subsequent allocate request returns delta
    allocateRequest1 =
        AllocateRequest.newInstance(response1.getResponseId(), 0F, null, null,
          null);
    response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1);
    updatedNodes = response1.getUpdatedNodes();
    Assert.assertEquals(1, updatedNodes.size());
    nr = updatedNodes.iterator().next();
    Assert.assertEquals(nm3.getNodeId(), nr.getNodeId());
    Assert.assertEquals(NodeState.LOST, nr.getNodeState());
        
    // registering another AM gives it the complete failed list
    RMApp app2 = rm.submitApp(2000);
    // Trigger nm2 heartbeat so that AM gets launched on it
    nm2.nodeHeartbeat(true);
    RMAppAttempt attempt2 = app2.getCurrentAppAttempt();
    MockAM am2 = rm.sendAMLaunched(attempt2.getAppAttemptId());
    
    // register AM returns all unusable nodes
    am2.registerAppAttempt();
    
    // allocate request returns no updated node
    AllocateRequest allocateRequest2 =
        AllocateRequest.newInstance(0, 0F, null, null, null);
    AllocateResponse response2 =
        allocate(attempt2.getAppAttemptId(), allocateRequest2);
    updatedNodes = response2.getUpdatedNodes();
    Assert.assertEquals(0, updatedNodes.size());
    
    syncNodeHeartbeat(nm4, true);
    
    // both AM's should get delta updated nodes
    allocateRequest1 =
        AllocateRequest.newInstance(response1.getResponseId(), 0F, null, null,
          null);
    response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1);
    updatedNodes = response1.getUpdatedNodes();
    Assert.assertEquals(1, updatedNodes.size());
    nr = updatedNodes.iterator().next();
    Assert.assertEquals(nm4.getNodeId(), nr.getNodeId());
    Assert.assertEquals(NodeState.RUNNING, nr.getNodeState());
    
    allocateRequest2 =
        AllocateRequest.newInstance(response2.getResponseId(), 0F, null, null,
          null);
    response2 = allocate(attempt2.getAppAttemptId(), allocateRequest2);
    updatedNodes = response2.getUpdatedNodes();
    Assert.assertEquals(1, updatedNodes.size());
    nr = updatedNodes.iterator().next();
    Assert.assertEquals(nm4.getNodeId(), nr.getNodeId());
    Assert.assertEquals(NodeState.RUNNING, nr.getNodeState());


    // subsequent allocate calls should return no updated nodes
    allocateRequest2 =
        AllocateRequest.newInstance(response2.getResponseId(), 0F, null, null,
          null);
    response2 = allocate(attempt2.getAppAttemptId(), allocateRequest2);
    updatedNodes = response2.getUpdatedNodes();
    Assert.assertEquals(0, updatedNodes.size());
    
    // how to do the above for LOST node
  
  }

View Full Code Here

Examples of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

    List<NMToken> expectedNMTokens = new ArrayList<NMToken>();


    // am1 allocate 2 container on nm1.
    // first container
    while (true) {
      AllocateResponse response =
          am1.allocate("127.0.0.1", 2000, 2,
            new ArrayList<ContainerId>());
      nm1.nodeHeartbeat(true);
      containers.addAll(response.getAllocatedContainers());
      expectedNMTokens.addAll(response.getNMTokens());
      if (containers.size() == 2) {
        break;
      }
      Thread.sleep(200);
      System.out.println("Waiting for container to be allocated.");
    }
    // launch the container-2
    nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.RUNNING);
    ContainerId containerId2 =
        ContainerId.newInstance(am1.getApplicationAttemptId(), 2);
    rm1.waitForState(nm1, containerId2, RMContainerState.RUNNING);
    // launch the container-3
    nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 3, ContainerState.RUNNING);
    ContainerId containerId3 =
        ContainerId.newInstance(am1.getApplicationAttemptId(), 3);
    rm1.waitForState(nm1, containerId3, RMContainerState.RUNNING);
    
    // fail am1
    nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
    am1.waitForState(RMAppAttemptState.FAILED);
    rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);


    // restart the am
    MockAM am2 = MockRM.launchAM(app1, rm1, nm1);
    RegisterApplicationMasterResponse registerResponse =
        am2.registerAppAttempt();
    rm1.waitForState(app1.getApplicationId(), RMAppState.RUNNING);


    // check am2 get the nm token from am1.
    Assert.assertEquals(expectedNMTokens,
      registerResponse.getNMTokensFromPreviousAttempts());


    // am2 allocate 1 container on nm2
    containers = new ArrayList<Container>();
    while (true) {
      AllocateResponse allocateResponse =
          am2.allocate("127.1.1.1", 4000, 1,
            new ArrayList<ContainerId>());
      nm2.nodeHeartbeat(true);
      containers.addAll(allocateResponse.getAllocatedContainers());
      expectedNMTokens.addAll(allocateResponse.getNMTokens());
      if (containers.size() == 1) {
        break;
      }
      Thread.sleep(200);
      System.out.println("Waiting for container to be allocated.");

View Full Code Here

Examples of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

    // nmTokens keeps track of all the nmTokens issued in the allocate call.
    List<NMToken> expectedNMTokens = new ArrayList<NMToken>();


    // am1 allocate 1 container on nm1.
    while (true) {
      AllocateResponse response =
          am.allocate("127.0.0.1", 2000, NUM_CONTAINERS,
            new ArrayList<ContainerId>());
      nm1.nodeHeartbeat(true);
      containers.addAll(response.getAllocatedContainers());
      expectedNMTokens.addAll(response.getNMTokens());
      if (containers.size() == NUM_CONTAINERS) {
        break;
      }
      Thread.sleep(200);
      System.out.println("Waiting for container to be allocated.");

View Full Code Here

Examples of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

      List<ContainerId> releaseContainerList =
          new ArrayList<ContainerId>();
      HashMap<String, Token> nmTokens = new HashMap<String, Token>();


      // initially requesting 2 containers.
      AllocateResponse response =
          am.allocate("h1", 1000, 2, releaseContainerList);
      Assert.assertEquals(0, response.getAllocatedContainers().size());
      allocateContainersAndValidateNMTokens(am, containersReceivedForNM1, 2,
          nmTokens, nm1);
      Assert.assertEquals(1, nmTokens.size());


      
      // requesting 2 more containers.
      response = am.allocate("h1", 1000, 2, releaseContainerList);
      Assert.assertEquals(0, response.getAllocatedContainers().size());
      allocateContainersAndValidateNMTokens(am, containersReceivedForNM1, 4,
          nmTokens, nm1);
      Assert.assertEquals(1, nmTokens.size());
      
      
      // We will be simulating NM restart so restarting newly added h2:1234
      // NM 2 now registers.
      MockNM nm2 = rm.registerNode("h2:1234", 10000);
      nm2.nodeHeartbeat(true);
      ArrayList<Container> containersReceivedForNM2 =
          new ArrayList<Container>();
      
      response = am.allocate("h2", 1000, 2, releaseContainerList);
      Assert.assertEquals(0, response.getAllocatedContainers().size());
      allocateContainersAndValidateNMTokens(am, containersReceivedForNM2, 2,
          nmTokens, nm2);
      Assert.assertEquals(2, nmTokens.size());
      
      // Simulating NM-2 restart.
      nm2 = rm.registerNode("h2:1234", 10000);
      // Wait for reconnect to make it through the RM and create a new RMNode
      Map<NodeId, RMNode> nodes = rm.getRMContext().getRMNodes();
      while (nodes.get(nm2.getNodeId()).getLastNodeHeartBeatResponse()
          .getResponseId() > 0) {
        Thread.sleep(WAIT_SLEEP_MS);
      }


      int interval = 40;
      // Wait for nm Token to be cleared.
      while (nmTokenSecretManager
          .isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(),
              nm2.getNodeId()) && interval-- > 0) {
        LOG.info("waiting for nmToken to be cleared for : " + nm2.getNodeId());
        Thread.sleep(WAIT_SLEEP_MS);
      }
      Assert.assertTrue(nmTokenSecretManager
          .isApplicationAttemptRegistered(attempt.getAppAttemptId()));
      
      // removing NMToken for h2:1234
      nmTokens.remove(nm2.getNodeId().toString());
      Assert.assertEquals(1, nmTokens.size());
      
      // We should again receive the NMToken.
      response = am.allocate("h2", 1000, 2, releaseContainerList);
      Assert.assertEquals(0, response.getAllocatedContainers().size());
      allocateContainersAndValidateNMTokens(am, containersReceivedForNM2, 4,
          nmTokens, nm2);
      Assert.assertEquals(2, nmTokens.size());


      // Now rolling over NMToken masterKey. it should resend the NMToken in
      // next allocate call.
      Assert.assertTrue(nmTokenSecretManager
          .isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(),
              nm1.getNodeId()));
      Assert.assertTrue(nmTokenSecretManager
          .isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(),
              nm2.getNodeId()));
      
      nmTokenSecretManager.rollMasterKey();
      nmTokenSecretManager.activateNextMasterKey();
      
      Assert.assertFalse(nmTokenSecretManager
          .isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(),
              nm1.getNodeId()));
      Assert.assertFalse(nmTokenSecretManager
          .isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(),
              nm2.getNodeId()));
      // It should not remove application attempt entry.
      Assert.assertTrue(nmTokenSecretManager
          .isApplicationAttemptRegistered(attempt.getAppAttemptId()));


      nmTokens.clear();
      Assert.assertEquals(0, nmTokens.size());
      // We should again receive the NMToken.
      response = am.allocate("h2", 1000, 1, releaseContainerList);
      Assert.assertEquals(0, response.getAllocatedContainers().size());
      allocateContainersAndValidateNMTokens(am, containersReceivedForNM2, 5,
          nmTokens, nm2);
      Assert.assertEquals(1, nmTokens.size());
      Assert.assertTrue(nmTokenSecretManager
          .isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(),

View Full Code Here

Examples of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

  protected void allocateContainersAndValidateNMTokens(MockAM am,
      ArrayList<Container> containersReceived, int totalContainerRequested,
      HashMap<String, Token> nmTokens, MockNM nm) throws Exception,
      InterruptedException {
    ArrayList<ContainerId> releaseContainerList = new ArrayList<ContainerId>();
    AllocateResponse response;
    ArrayList<ResourceRequest> resourceRequest =
        new ArrayList<ResourceRequest>();      
    while (containersReceived.size() < totalContainerRequested) {
      nm.nodeHeartbeat(true);
      LOG.info("requesting containers..");
      response =
          am.allocate(resourceRequest, releaseContainerList);
      containersReceived.addAll(response.getAllocatedContainers());
      if (!response.getNMTokens().isEmpty()) {
        for (NMToken nmToken : response.getNMTokens()) {
          String nodeId = nmToken.getNodeId().toString();
          if (nmTokens.containsKey(nodeId)) {
            Assert.fail("Duplicate NMToken received for : " + nodeId);
          }
          nmTokens.put(nodeId, nmToken.getToken());

View Full Code Here

Examples of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

  public static AllocateResponse newAllocateResponse(int responseId,
      List<ContainerStatus> completedContainers,
      List<Container> allocatedContainers, List<NodeReport> updatedNodes,
      Resource availResources, AMCommand command, int numClusterNodes,
      PreemptionMessage preempt) {
    AllocateResponse response = recordFactory
        .newRecordInstance(AllocateResponse.class);
    response.setNumClusterNodes(numClusterNodes);
    response.setResponseId(responseId);
    response.setCompletedContainersStatuses(completedContainers);
    response.setAllocatedContainers(allocatedContainers);
    response.setUpdatedNodes(updatedNodes);
    response.setAvailableResources(availResources);
    response.setAMCommand(command);
    response.setPreemptionMessage(preempt);


    return response;
  }

View Full Code Here

Examples of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

    Assert.assertEquals(1, loadedApp2.getAppAttempts().size());
    
    // verify old AM is not accepted
    // change running AM to talk to new RM
    am1.setAMRMProtocol(rm2.getApplicationMasterService());
    AllocateResponse allocResponse = am1.allocate(
        new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>());
    Assert.assertEquals(AMCommand.AM_SHUTDOWN, allocResponse.getAMCommand());
    
    // NM should be rebooted on heartbeat, even first heartbeat for nm2
    NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true);
    Assert.assertEquals(NodeAction.RESYNC, hbResponse.getNodeAction());
    hbResponse = nm2.nodeHeartbeat(true);

View Full Code Here

Examples of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
    MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
    am1.registerAppAttempt();


    am1.addRequests(new String[] { "127.0.0.1" }, GB, 1, 1);
    AllocateResponse alloc1Response = am1.schedule(); // send the request


    // kick the scheduler
    nm1.nodeHeartbeat(true);
    while (alloc1Response.getAllocatedContainers().size() < 1) {
      LOG.info("Waiting for containers to be created for app 1...");
      sleep(1000);
      alloc1Response = am1.schedule();
    }


    // assert RMIdentifer is set properly in allocated containers
    Container allocatedContainer =
        alloc1Response.getAllocatedContainers().get(0);
    ContainerTokenIdentifier tokenId =
        BuilderUtils.newContainerTokenIdentifier(allocatedContainer
          .getContainerToken());
    Assert.assertEquals(MockRM.getClusterTimeStamp(), tokenId.getRMIdentifer());
    rm.stop();

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.