Examples of AMResponse


Examples of org.apache.hadoop.yarn.api.records.AMResponse

    LOG.info("Registering " + attemptId);
    responseMap.put(attemptId, response);
  }

  public void unregisterAttempt(ApplicationAttemptId attemptId) {
    AMResponse lastResponse = responseMap.get(attemptId);
    if (lastResponse != null) {
      synchronized (lastResponse) {
        responseMap.remove(attemptId);
      }
    }
View Full Code Here

Examples of org.apache.hadoop.yarn.api.records.AMResponse

    AllocateRequest allocateRequest = BuilderUtils.newAllocateRequest(
        applicationAttemptId, lastResponseID, super.getApplicationProgress(),
        new ArrayList<ResourceRequest>(ask), new ArrayList<ContainerId>(
            release));
    AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
    AMResponse response = allocateResponse.getAMResponse();
    lastResponseID = response.getResponseId();
    availableResources = response.getAvailableResources();
    lastClusterNmCount = clusterNmCount;
    clusterNmCount = allocateResponse.getNumClusterNodes();

    if (ask.size() > 0 || release.size() > 0) {
      LOG.info("getResources() for " + applicationId + ":" + " ask="
          + ask.size() + " release= " + release.size() + " newContainers="
          + response.getAllocatedContainers().size() + " finishedContainers="
          + response.getCompletedContainersStatuses().size()
          + " resourcelimit=" + availableResources + " knownNMs="
          + clusterNmCount);
    }

    ask.clear();
View Full Code Here

Examples of org.apache.hadoop.yarn.api.records.AMResponse

  }
 
  @SuppressWarnings("unchecked")
  private List<Container> getResources() throws Exception {
    int headRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;//first time it would be null
    AMResponse response;
    /*
     * If contact with RM is lost, the AM will wait MR_AM_TO_RM_WAIT_INTERVAL_MS
     * milliseconds before aborting. During this interval, AM will still try
     * to contact the RM.
     */
    try {
      response = makeRemoteRequest();
      // Reset retry count if no exception occurred.
      retrystartTime = System.currentTimeMillis();
    } catch (Exception e) {
      // This can happen when the connection to the RM has gone down. Keep
      // re-trying until the retryInterval has expired.
      if (System.currentTimeMillis() - retrystartTime >= retryInterval) {
        LOG.error("Could not contact RM after " + retryInterval + " milliseconds.");
        eventHandler.handle(new JobEvent(this.getJob().getID(),
                                         JobEventType.INTERNAL_ERROR));
        throw new YarnException("Could not contact RM after " +
                                retryInterval + " milliseconds.");
      }
      // Throw this up to the caller, which may decide to ignore it and
      // continue to attempt to contact the RM.
      throw e;
    }
    if (response.getReboot()) {
      // This can happen if the RM has been restarted. If it is in that state,
      // this application must clean itself up.
      eventHandler.handle(new JobEvent(this.getJob().getID(),
                                       JobEventType.INTERNAL_ERROR));
      throw new YarnException("Resource Manager doesn't recognize AttemptId: " +
                               this.getContext().getApplicationID());
    }
    int newHeadRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;
    List<Container> newContainers = response.getAllocatedContainers();
    List<ContainerStatus> finishedContainers = response.getCompletedContainersStatuses();
    if (newContainers.size() + finishedContainers.size() > 0 || headRoom != newHeadRoom) {
      //something changed
      recalculateReduceSchedule = true;
      if (LOG.isDebugEnabled() && headRoom != newHeadRoom) {
        LOG.debug("headroom=" + newHeadRoom);
View Full Code Here

Examples of org.apache.hadoop.yarn.api.records.AMResponse

                        "host" + containerId.getId() + ":5678", req
                          .getCapability(), req.getPriority(), null));
                  }
                }

                AMResponse amResponse = Records.newRecord(AMResponse.class);
                amResponse.setAllocatedContainers(containers);
                amResponse.setResponseId(request.getResponseId() + 1);
                response.setAMResponse(amResponse);
                response.setNumClusterNodes(350);
                return response;
              }
            };
View Full Code Here

Examples of org.apache.hadoop.yarn.api.records.AMResponse

      int containers) throws Exception {
    requests.addAll(createReq(hosts, memory, priority, containers));
  }

  public AMResponse schedule() throws Exception {
    AMResponse response = allocate(requests, releases);
    requests.clear();
    releases.clear();
    return response;
  }
View Full Code Here

Examples of org.apache.hadoop.yarn.api.records.AMResponse

    ApplicationAttemptId applicationAttemptId = request
        .getApplicationAttemptId();
    authorizeRequest(applicationAttemptId);

    ApplicationId appID = applicationAttemptId.getApplicationId();
    AMResponse lastResponse = responseMap.get(applicationAttemptId);
    if (lastResponse == null) {
      String message = "Application doesn't exist in cache "
          + applicationAttemptId;
      LOG.error(message);
      RMAuditLogger.logFailure(this.rmContext.getRMApps().get(appID).getUser(),
View Full Code Here

Examples of org.apache.hadoop.yarn.api.records.AMResponse

    ApplicationAttemptId applicationAttemptId = request
        .getApplicationAttemptId();
    authorizeRequest(applicationAttemptId);

    AMResponse lastResponse = responseMap.get(applicationAttemptId);
    if (lastResponse == null) {
      String message = "Application doesn't exist in cache "
          + applicationAttemptId;
      LOG.error(message);
      throw RPCUtil.getRemoteException(message);
View Full Code Here

Examples of org.apache.hadoop.yarn.api.records.AMResponse

    this.amLivelinessMonitor.receivedPing(appAttemptId);

    /* check if its in cache */
    AllocateResponse allocateResponse = recordFactory
        .newRecordInstance(AllocateResponse.class);
    AMResponse lastResponse = responseMap.get(appAttemptId);
    if (lastResponse == null) {
      LOG.error("AppAttemptId doesnt exist in cache " + appAttemptId);
      allocateResponse.setAMResponse(reboot);
      return allocateResponse;
    }
    if ((request.getResponseId() + 1) == lastResponse.getResponseId()) {
      /* old heartbeat */
      allocateResponse.setAMResponse(lastResponse);
      return allocateResponse;
    } else if (request.getResponseId() + 1 < lastResponse.getResponseId()) {
      LOG.error("Invalid responseid from appAttemptId " + appAttemptId);
      // Oh damn! Sending reboot isn't enough. RM state is corrupted. TODO:
      // Reboot is not useful since after AM reboots, it will send register and
      // get an exception. Might as well throw an exception here.
      allocateResponse.setAMResponse(reboot);
      return allocateResponse;
    }
   
    // Allow only one thread in AM to do heartbeat at a time.
    synchronized (lastResponse) {

      // Send the status update to the appAttempt.
      this.rmContext.getDispatcher().getEventHandler().handle(
          new RMAppAttemptStatusupdateEvent(appAttemptId, request
              .getProgress()));

      List<ResourceRequest> ask = request.getAskList();
      List<ContainerId> release = request.getReleaseList();

      // Send new requests to appAttempt.
      Allocation allocation =
          this.rScheduler.allocate(appAttemptId, ask, release);

      RMApp app = this.rmContext.getRMApps().get(
          appAttemptId.getApplicationId());
      RMAppAttempt appAttempt = app.getRMAppAttempt(appAttemptId);
     
      AMResponse response = recordFactory.newRecordInstance(AMResponse.class);

      // update the response with the deltas of node status changes
      List<RMNode> updatedNodes = new ArrayList<RMNode>();
      if(app.pullRMNodeUpdates(updatedNodes) > 0) {
        List<NodeReport> updatedNodeReports = new ArrayList<NodeReport>();
        for(RMNode rmNode: updatedNodes) {
          SchedulerNodeReport schedulerNodeReport = 
              rScheduler.getNodeReport(rmNode.getNodeID());
          Resource used = BuilderUtils.newResource(0, 0);
          int numContainers = 0;
          if (schedulerNodeReport != null) {
            used = schedulerNodeReport.getUsedResource();
            numContainers = schedulerNodeReport.getNumContainers();
          }
          NodeReport report = BuilderUtils.newNodeReport(rmNode.getNodeID(),
              rmNode.getState(),
              rmNode.getHttpAddress(), rmNode.getRackName(), used,
              rmNode.getTotalCapability(), numContainers,
              rmNode.getNodeHealthStatus());
         
          updatedNodeReports.add(report);
        }
        response.setUpdatedNodes(updatedNodeReports);
      }

      response.setAllocatedContainers(allocation.getContainers());
      response.setCompletedContainersStatuses(appAttempt
          .pullJustFinishedContainers());
      response.setResponseId(lastResponse.getResponseId() + 1);
      response.setAvailableResources(allocation.getResourceLimit());
     
      AMResponse oldResponse = responseMap.put(appAttemptId, response);
      if (oldResponse == null) {
        // appAttempt got unregistered, remove it back out
        responseMap.remove(appAttemptId);
        String message = "App Attempt removed from the cache during allocate"
            + appAttemptId;
View Full Code Here

Examples of org.apache.hadoop.yarn.api.records.AMResponse

      return allocateResponse;
    }
  }

  public void registerAppAttempt(ApplicationAttemptId attemptId) {
    AMResponse response = recordFactory.newRecordInstance(AMResponse.class);
    response.setResponseId(0);
    LOG.info("Registering " + attemptId);
    responseMap.put(attemptId, response);
  }
View Full Code Here

Examples of org.apache.hadoop.yarn.api.records.AMResponse

          resourceManager.addContainerRequest(containerAsk);
        }

        // Send the request to RM
        LOG.info("Asking RM for containers" + ", askCount=" + askCount);
        AMResponse amResp = sendContainerAskToRM();

        // Retrieve list of allocated containers from the response
        List<Container> allocatedContainers = amResp.getAllocatedContainers();
        LOG.info("Got response from RM for container ask, allocatedCnt="
            + allocatedContainers.size());
        numAllocatedContainers.addAndGet(allocatedContainers.size());
        for (Container allocatedContainer : allocatedContainers) {
          LOG.info("Launching shell command on a new container."
              + ", containerId=" + allocatedContainer.getId()
              + ", containerNode=" + allocatedContainer.getNodeId().getHost()
              + ":" + allocatedContainer.getNodeId().getPort()
              + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress()
              + ", containerState" + allocatedContainer.getState()
              + ", containerResourceMemory"
              + allocatedContainer.getResource().getMemory());
          // + ", containerToken"
          // +allocatedContainer.getContainerToken().getIdentifier().toString());

          LaunchContainerRunnable runnableLaunchContainer = new LaunchContainerRunnable(
              allocatedContainer);
          Thread launchThread = new Thread(runnableLaunchContainer);

          // launch and start the container on a separate thread to keep
          // the main thread unblocked
          // as all containers may not be allocated at one go.
          launchThreads.add(launchThread);
          launchThread.start();
        }

        // Check what the current available resources in the cluster are
        // TODO should we do anything if the available resources are not enough?
        Resource availableResources = amResp.getAvailableResources();
        LOG.info("Current available resources in the cluster "
            + availableResources);

        // Check the completed containers
        List<ContainerStatus> completedContainers = amResp
            .getCompletedContainersStatuses();
        LOG.info("Got response from RM for container ask, completedCnt="
            + completedContainers.size());
        for (ContainerStatus containerStatus : completedContainers) {
          LOG.info("Got container status for containerID="
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.