Examples of TaskTracker


Examples of org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker

      synchronized (JobTracker.this) {
        synchronized (taskTrackers) {
          synchronized (trackerExpiryQueue) {
            // IV. Register a new tracker
            TaskTracker taskTracker = getTaskTracker(trackerName);
            boolean isTrackerRegistered =  (taskTracker != null);
            if (!isTrackerRegistered) {
              markTracker(trackerName); // add the tracker to recovery-manager
              taskTracker = new TaskTracker(trackerName);
              taskTracker.setStatus(ttStatus);
              addNewTracker(taskTracker);
            }
     
            // V. Update the tracker status
            // This will update the meta info of the jobtracker and also add the
View Full Code Here

Examples of org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker

    public FakeTaskTrackerManager() {
      JobConf conf = new JobConf();
      queueManager = new QueueManager(conf);
     
      TaskTracker tt1 = new TaskTracker("tt1");
      tt1.setStatus(new TaskTrackerStatus("tt1", "tt1.host", 1,
                    new ArrayList<TaskStatus>(), 0,
                    maxMapTasksPerTracker, maxReduceTasksPerTracker));
      trackers.put("tt1", tt1);
     
      TaskTracker tt2 = new TaskTracker("tt2");
      tt2.setStatus(new TaskTrackerStatus("tt2", "tt2.host", 2,
                    new ArrayList<TaskStatus>(), 0,
                    maxMapTasksPerTracker, maxReduceTasksPerTracker));
      trackers.put("tt2", tt2);
    }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker

   
    boolean change = tip.updateStatus(status);
    if (change) {
      TaskStatus.State state = status.getRunState();
      // get the TaskTrackerStatus where the task ran
      TaskTracker taskTracker =
        this.jobtracker.getTaskTracker(tip.machineWhereTaskRan(taskid));
      TaskTrackerStatus ttStatus =
        (taskTracker == null) ? null : taskTracker.getStatus();
      String httpTaskLogLocation = null;

      if (null != ttStatus){
        String host;
        if (NetUtils.getStaticResolution(ttStatus.getHost()) != null) {
View Full Code Here

Examples of org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker

    return false;
  }

  // lock to taskTrackers should hold JT lock first.
  synchronized public TaskTrackerStatus getTaskTrackerStatus(String trackerID) {
    TaskTracker taskTracker;
    synchronized (taskTrackers) {
      taskTracker = taskTrackers.get(trackerID);
    }
    return (taskTracker == null) ? null : taskTracker.getStatus();
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker

   * @param status The new status for the task tracker
   * @return Was an old status found?
   */
  boolean updateTaskTrackerStatus(String trackerName,
                                  TaskTrackerStatus status) {
    TaskTracker tt = getTaskTracker(trackerName);
    TaskTrackerStatus oldStatus = (tt == null) ? null : tt.getStatus();

    // update the total cluster capacity first

    if (status != null) {
      // we have a fresh tasktracker status
      if (!faultyTrackers.isBlacklisted(status.getHost())) {
        // if the task tracker host is not blacklisted - then
        // we update the cluster capacity with the capacity
        // reported by the tasktracker
        updateTotalTaskCapacity(status);
      } else {
        // if the tasktracker is blacklisted - then it's capacity
        // is already removed and will only be added back to the
        // cluster capacity when it's unblacklisted
      }
    } else {
      if (oldStatus != null) {
        // old status exists - but no new status. in this case
        // we are removing the tracker from the cluster.

        if (!faultyTrackers.isBlacklisted(oldStatus.getHost())) {

          // we update the total task capacity based on the old status
          // this seems redundant - but this call is idempotent - so just
          // make it. the danger of not making it is that we may accidentally
          // remove something we never had
          updateTotalTaskCapacity(oldStatus);
          removeTaskTrackerCapacity(oldStatus);

        } else {
          // if the host is blacklisted - then the tracker's capacity
          // has already been removed and there's nothing to do
        }
      }
    }


    if (oldStatus != null) {
      totalMaps -= oldStatus.countMapTasks();
      totalReduces -= oldStatus.countReduceTasks();
      occupiedMapSlots -= oldStatus.countOccupiedMapSlots();
      occupiedReduceSlots -= oldStatus.countOccupiedReduceSlots();
      getInstrumentation().decRunningMaps(oldStatus.countMapTasks());
      getInstrumentation().decRunningReduces(oldStatus.countReduceTasks());
      getInstrumentation().decOccupiedMapSlots(oldStatus.countOccupiedMapSlots());
      getInstrumentation().decOccupiedReduceSlots(oldStatus.countOccupiedReduceSlots());
      if (status == null) {
        taskTrackers.remove(trackerName);
        Integer numTaskTrackersInHost =
          uniqueHostsMap.get(oldStatus.getHost());
        if (numTaskTrackersInHost != null) {
          numTaskTrackersInHost --;
          if (numTaskTrackersInHost > 0)  {
            uniqueHostsMap.put(oldStatus.getHost(), numTaskTrackersInHost);
          }
          else {
            uniqueHostsMap.remove(oldStatus.getHost());
          }
        }
      }
    }
    if (status != null) {
      totalMaps += status.countMapTasks();
      totalReduces += status.countReduceTasks();
      occupiedMapSlots += status.countOccupiedMapSlots();
      occupiedReduceSlots += status.countOccupiedReduceSlots();
      getInstrumentation().addRunningMaps(status.countMapTasks());
      getInstrumentation().addRunningReduces(status.countReduceTasks());
      getInstrumentation().addOccupiedMapSlots(status.countOccupiedMapSlots());
      getInstrumentation().addOccupiedReduceSlots(status.countOccupiedReduceSlots());

      boolean alreadyPresent = false;
      TaskTracker taskTracker = taskTrackers.get(trackerName);
      if (taskTracker != null) {
        alreadyPresent = true;
      } else {
        taskTracker = new TaskTracker(trackerName);
      }

      taskTracker.setStatus(status);
      taskTrackers.put(trackerName, taskTracker);

      if (LOG.isDebugEnabled()) {
        int runningMaps = 0, runningReduces = 0;
        int commitPendingMaps = 0, commitPendingReduces = 0;
View Full Code Here

Examples of org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker

    synchronized (taskTrackers) {
      synchronized (trackerExpiryQueue) {
        boolean seenBefore = updateTaskTrackerStatus(trackerName,
                                                     trackerStatus);
        TaskTracker taskTracker = getTaskTracker(trackerName);
        if (initialContact) {
          // If it's first contact, then clear out
          // any state hanging around
          if (seenBefore) {
            LOG.warn("initialContact but seenBefore from Tracker : " + trackerName);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker

                  // Remove profile from head of queue
                  trackerExpiryQueue.remove(leastRecent);
                  String trackerName = leastRecent.getTrackerName();

                  // Figure out if last-seen time should be updated, or if tracker is dead
                  TaskTracker current = getTaskTracker(trackerName);
                  TaskTrackerStatus newProfile =
                    (current == null ) ? null : current.getStatus();
                  // Items might leave the taskTracker set through other means; the
                  // status stored in 'taskTrackers' might be null, which means the
                  // tracker has already been destroyed.
                  if (newProfile != null) {
                    if ((now - newProfile.getLastSeen()) > TASKTRACKER_EXPIRY_INTERVAL) {
View Full Code Here

Examples of org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker

    boolean change = tip.updateStatus(status);
    if (change) {
      TaskStatus.State state = status.getRunState();
      // get the TaskTrackerStatus where the task ran
      TaskTracker taskTracker =
        this.jobtracker.getTaskTracker(tip.machineWhereTaskRan(taskid));
      TaskTrackerStatus ttStatus =
        (taskTracker == null) ? null : taskTracker.getStatus();
      String httpTaskLogLocation = null;

      if (null != ttStatus){
        String host;
        if (NetUtils.getStaticResolution(ttStatus.getHost()) != null) {
View Full Code Here

Examples of org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker

            trackerExpiryQueue.remove(leastRecent);
            String trackerName = leastRecent.getTrackerName();

            // Figure out if last-seen time should be updated, or if
            // tracker is dead
            TaskTracker current = getTaskTracker(trackerName);
            TaskTrackerStatus newProfile =
              (current == null ) ? null : current.getStatus();
            // Items might leave the taskTracker set through other means; the
            // status stored in 'taskTrackers' might be null, which means the
            // tracker has already been destroyed.
            if (newProfile != null) {
              if ((now - newProfile.getLastSeen()) >
View Full Code Here

Examples of org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker

    return false;
  }
 
  // lock to taskTrackers should hold JT lock first.
  synchronized public TaskTrackerStatus getTaskTrackerStatus(String trackerID) {
    TaskTracker taskTracker;
    synchronized (taskTrackers) {
      taskTracker = taskTrackers.get(trackerID);
    }
    return (taskTracker == null) ? null : taskTracker.getStatus();
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.