Examples of org.apache.hadoop.yarn.server.api.records.HeartbeatResponse

Package org.apache.hadoop.yarn.server.api.records

Examples of org.apache.hadoop.yarn.server.api.records.HeartbeatResponse

org.apache.hadoop.yarn.server.api.records.HeartbeatResponse

        new ArrayList<ContainerStatus>();
    containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(0)
      .getId(), ContainerState.RUNNING, "nothing", 0));
    containerStatuses.put(app.getApplicationId(), containerStatusList);


    HeartbeatResponse resp = nm1.nodeHeartbeat(containerStatuses, true);
    dispatcher.await();
    List<ContainerId> contsToClean = resp.getContainersToCleanupList();
    int cleanedConts = contsToClean.size();
    waitCount = 0;
    while (cleanedConts < 1 && waitCount++ < 200) {
      LOG.info("Waiting to get cleanup events.. cleanedConts: " + cleanedConts);
      Thread.sleep(100);
      resp = nm1.nodeHeartbeat(true);
      dispatcher.await();
      contsToClean = resp.getContainersToCleanupList();
      cleanedConts += contsToClean.size();
    }
    LOG.info("Got cleanup for " + contsToClean.get(0));
    Assert.assertEquals(1, cleanedConts);


    // Now to test the case when RM already gave cleanup, and NM suddenly
    // realizes that the container is running.
    LOG.info("Testing container launch much after release and "
        + "NM getting cleanup");
    containerStatuses.clear();
    containerStatusList.clear();
    containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(0)
      .getId(), ContainerState.RUNNING, "nothing", 0));
    containerStatuses.put(app.getApplicationId(), containerStatusList);


    resp = nm1.nodeHeartbeat(containerStatuses, true);
    dispatcher.await();
    contsToClean = resp.getContainersToCleanupList();
    cleanedConts = contsToClean.size();
    // The cleanup list won't be instantaneous as it is given out by scheduler
    // and not RMNodeImpl.
    waitCount = 0;
    while (cleanedConts < 1 && waitCount++ < 200) {
      LOG.info("Waiting to get cleanup events.. cleanedConts: " + cleanedConts);
      Thread.sleep(100);
      resp = nm1.nodeHeartbeat(true);
      dispatcher.await();
      contsToClean = resp.getContainersToCleanupList();
      cleanedConts += contsToClean.size();
    }
    LOG.info("Got cleanup for " + contsToClean.get(0));
    Assert.assertEquals(1, cleanedConts);

View Full Code Here

  @After
  public void tearDown() throws Exception {
  }
  
  private RMNodeStatusEvent getMockRMNodeStatusEvent() {
    HeartbeatResponse response = mock(HeartbeatResponse.class);


    NodeHealthStatus healthStatus = mock(NodeHealthStatus.class);
    Boolean yes = new Boolean(true);
    doReturn(yes).when(healthStatus).getIsNodeHealthy();

View Full Code Here

    // but updating heartbeat response for cleanup does
    RMNodeStatusEvent statusEvent = getMockRMNodeStatusEvent();
    node.handle(statusEvent);
    Assert.assertEquals(1, node.getContainersToCleanUp().size());
    Assert.assertEquals(1, node.getAppsToCleanup().size());
    HeartbeatResponse hbrsp = Records.newRecord(HeartbeatResponse.class);
    node.updateHeartbeatResponseForCleanup(hbrsp);
    Assert.assertEquals(0, node.getContainersToCleanUp().size());
    Assert.assertEquals(0, node.getAppsToCleanup().size());
    Assert.assertEquals(1, hbrsp.getContainersToCleanupCount());
    Assert.assertEquals(completedContainerId, hbrsp.getContainerToCleanup(0));
    Assert.assertEquals(1, hbrsp.getApplicationsToCleanupCount());
    Assert.assertEquals(finishedAppId, hbrsp.getApplicationsToCleanup(0));
  }

View Full Code Here


    NodeHeartbeatResponse nodeHeartBeatResponse = recordFactory
        .newRecordInstance(NodeHeartbeatResponse.class);
    
    // 3. Check if it's a 'fresh' heartbeat i.e. not duplicate heartbeat
    HeartbeatResponse lastHeartbeatResponse = rmNode.getLastHeartBeatResponse();
    if (remoteNodeStatus.getResponseId() + 1 == lastHeartbeatResponse
        .getResponseId()) {
      LOG.info("Received duplicate heartbeat from node "
          + rmNode.getNodeAddress());
      nodeHeartBeatResponse.setHeartbeatResponse(lastHeartbeatResponse);
      return nodeHeartBeatResponse;
    } else if (remoteNodeStatus.getResponseId() + 1 < lastHeartbeatResponse
        .getResponseId()) {
      LOG.info("Too far behind rm response id:"
          + lastHeartbeatResponse.getResponseId() + " nm response id:"
          + remoteNodeStatus.getResponseId());
      // TODO: Just sending reboot is not enough. Think more.
      this.rmContext.getDispatcher().getEventHandler().handle(
          new RMNodeEvent(nodeId, RMNodeEventType.REBOOTING));
      return reboot;
    }


    // Heartbeat response
    HeartbeatResponse latestResponse = recordFactory
        .newRecordInstance(HeartbeatResponse.class);
    latestResponse.setResponseId(lastHeartbeatResponse.getResponseId() + 1);
    latestResponse.addAllContainersToCleanup(rmNode.getContainersToCleanUp());
    latestResponse.addAllApplicationsToCleanup(rmNode.getAppsToCleanup());
    latestResponse.setNodeAction(NodeAction.NORMAL);


    // Check if node's masterKey needs to be updated and if the currentKey has
    // roller over, send it across
    if (isSecurityEnabled()) {


      boolean shouldSendMasterKey = false;


      MasterKey nextMasterKeyForNode =
          this.containerTokenSecretManager.getNextKey();
      if (nextMasterKeyForNode != null) {
        // nextMasterKeyForNode can be null if there is no outstanding key that
        // is in the activation period.
        MasterKey nodeKnownMasterKey = request.getLastKnownMasterKey();
        if (nodeKnownMasterKey.getKeyId() != nextMasterKeyForNode.getKeyId()) {
          shouldSendMasterKey = true;
        }
      }
      if (shouldSendMasterKey) {
        latestResponse.setMasterKey(nextMasterKeyForNode);
      }
    }


    // 4. Send status to RMNode, saving the latest response.
    this.rmContext.getDispatcher().getEventHandler().handle(

View Full Code Here

  @After
  public void tearDown() throws Exception {
  }
  
  private RMNodeStatusEvent getMockRMNodeStatusEvent() {
    HeartbeatResponse response = mock(HeartbeatResponse.class);


    NodeHealthStatus healthStatus = mock(NodeHealthStatus.class);
    Boolean yes = new Boolean(true);
    doReturn(yes).when(healthStatus).getIsNodeHealthy();

View Full Code Here

    AMResponse amResponse = am1.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>());
    Assert.assertTrue(amResponse.getReboot());
    
    // NM should be rebooted on heartbeat, even first heartbeat for nm2
    HeartbeatResponse hbResponse = nm1.nodeHeartbeat(true);
    Assert.assertEquals(NodeAction.REBOOT, hbResponse.getNodeAction());
    hbResponse = nm2.nodeHeartbeat(true);
    Assert.assertEquals(NodeAction.REBOOT, hbResponse.getNodeAction());
    
    // new NM to represent NM re-register
    nm1 = rm2.registerNode("h1:1234", 15120);
    nm2 = rm2.registerNode("h2:5678", 15120);


    // verify no more reboot response sent
    hbResponse = nm1.nodeHeartbeat(true);
    Assert.assertTrue(NodeAction.REBOOT != hbResponse.getNodeAction());
    hbResponse = nm2.nodeHeartbeat(true);
    Assert.assertTrue(NodeAction.REBOOT != hbResponse.getNodeAction());
    
    // assert app1 attempt is saved
    attempt1 = loadedApp1.getCurrentAppAttempt();
    attemptId1 = attempt1.getAppAttemptId();
    rm2.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);

View Full Code Here

            nodeStatus.setResponseId(lastHeartBeatID);
            
            NodeHeartbeatRequest request = recordFactory
                .newRecordInstance(NodeHeartbeatRequest.class);
            request.setNodeStatus(nodeStatus);            
            HeartbeatResponse response =
              resourceTracker.nodeHeartbeat(request).getHeartbeatResponse();
            if (response.getNodeAction() == NodeAction.SHUTDOWN) {
              LOG
                  .info("Recieved SHUTDOWN signal from Resourcemanager as part of heartbeat," +
                      " hence shutting down.");
              NodeStatusUpdaterImpl.this.stop();
              break;
            }
            if (response.getNodeAction() == NodeAction.REBOOT) {
              LOG.info("Node is out of sync with ResourceManager,"
                  + " hence shutting down.");
              NodeStatusUpdaterImpl.this.stop();
              break;
            }


            lastHeartBeatID = response.getResponseId();
            List<ContainerId> containersToCleanup = response
                .getContainersToCleanupList();
            if (containersToCleanup.size() != 0) {
              dispatcher.getEventHandler().handle(
                  new CMgrCompletedContainersEvent(containersToCleanup));
            }
            List<ApplicationId> appsToCleanup =
                response.getApplicationsToCleanupList();
            if (appsToCleanup.size() != 0) {
              dispatcher.getEventHandler().handle(
                  new CMgrCompletedAppsEvent(appsToCleanup));
            }
          } catch (Throwable e) {

View Full Code Here

    List<ApplicationId> apps = null;
    
    //currently only containers are cleaned via this
    //AM container is cleaned via container launcher
    while (cleanedConts < 2 || cleanedApps < 1) {
      HeartbeatResponse resp = nm1.nodeHeartbeat(true);
      contsToClean = resp.getContainersToCleanupList();
      apps = resp.getApplicationsToCleanupList();
      Log.info("Waiting to get cleanup events.. cleanedConts: "
          + cleanedConts + " cleanedApps: " + cleanedApps);
      cleanedConts += contsToClean.size();
      cleanedApps += apps.size();
      Thread.sleep(1000);

View Full Code Here

    @Override
    public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
        throws YarnRemoteException {
      NodeStatus nodeStatus = request.getNodeStatus();
      nodeStatus.setResponseId(heartBeatID++);
      HeartbeatResponse response = recordFactory
          .newRecordInstance(HeartbeatResponse.class);
      response.setResponseId(heartBeatID);
      response.setNodeAction(heartBeatNodeAction);
      
      NodeHeartbeatResponse nhResponse = recordFactory
      .newRecordInstance(NodeHeartbeatResponse.class);
      nhResponse.setHeartbeatResponse(response);
      return nhResponse;

View Full Code Here

        // Checks on the NM end
        ConcurrentMap<ContainerId, Container> activeContainers =
            this.context.getContainers();
        Assert.assertEquals(2, activeContainers.size());
      }
      HeartbeatResponse response = recordFactory
          .newRecordInstance(HeartbeatResponse.class);
      response.setResponseId(heartBeatID);
      
      NodeHeartbeatResponse nhResponse = recordFactory
          .newRecordInstance(NodeHeartbeatResponse.class);
      nhResponse.setHeartbeatResponse(response);
      return nhResponse;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.yarn.server.api.records.HeartbeatResponse

org.apache.hadoop.yarn.server.nodemanager.MockNodeStatusUpdater$MockResourceTracker

org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl

org.apache.hadoop.yarn.server.nodemanager.TestNodeStatusUpdater$MyResourceTracker

org.apache.hadoop.yarn.server.nodemanager.TestNodeStatusUpdater$MyResourceTracker2

org.apache.hadoop.yarn.server.nodemanager.TestNodeStatusUpdater$MyResourceTracker3

org.apache.hadoop.yarn.server.resourcemanager.MockNM

org.apache.hadoop.yarn.server.resourcemanager.NodeManager

org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.TestRMNMRPCResponseId

org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService

org.apache.hadoop.yarn.server.resourcemanager.TestApplicationCleanup

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.