Examples of org.apache.hadoop.mapred.MapTaskCompletionEventsUpdate

org.apache.hadoop.mapred.MapTaskCompletionEventsUpdate
A class that represents the communication between the tasktracker and child tasks w.r.t the map task completion events. It also indicates whether the child task should reset its events index.

   */  
  private int getMapCompletionEvents() throws IOException {
    
    int numNewMaps = 0;
    
    MapTaskCompletionEventsUpdate update = 
      umbilical.getMapCompletionEvents((org.apache.hadoop.mapred.JobID)
                                       reduce.getJobID(), 
                                       fromEventId, 
                                       MAX_EVENTS_TO_FETCH,
                                       (org.apache.hadoop.mapred.TaskAttemptID)
                                         reduce);
    TaskCompletionEvent events[] = update.getMapTaskCompletionEvents();
    LOG.debug("Got " + events.length + " map completion events from " + 
             fromEventId);
      
    // Check if the reset is required.
    // Since there is no ordering of the task completion events at the 
    // reducer, the only option to sync with the new jobtracker is to reset 
    // the events index
    if (update.shouldReset()) {
      fromEventId = 0;
      scheduler.resetKnownMaps();
    }
    
    // Update the last seen event ID

View Full Code Here

          eventIdx, true, TaskCompletionEvent.Status.SUCCEEDED,
          "http://somehost:8888");
      tceList.add(tce);
    }
    TaskCompletionEvent[] events = {};
    return new MapTaskCompletionEventsUpdate(tceList.toArray(events), false);
  }

View Full Code Here

    
    int numNewMaps = 0;
    TaskCompletionEvent events[] = null;


    do {
      MapTaskCompletionEventsUpdate update =
          umbilical.getMapCompletionEvents(
              (org.apache.hadoop.mapred.JobID)reduce.getJobID(),
              fromEventIdx,
              maxEventsToFetch,
              (org.apache.hadoop.mapred.TaskAttemptID)reduce);
      events = update.getMapTaskCompletionEvents();
      LOG.debug("Got " + events.length + " map completion events from " +
               fromEventIdx);


      // Check if the reset is required.
      // Since there is no ordering of the task completion events at the
      // reducer, the only option to sync with the new jobtracker is to reset
      // the events index
      if (update.shouldReset()) {
        fromEventIdx = 0;
        scheduler.resetKnownMaps();
      }


      // Update the last seen event ID

View Full Code Here

    
    int numNewMaps = 0;
    TaskCompletionEvent events[] = null;


    do {
      MapTaskCompletionEventsUpdate update =
          umbilical.getMapCompletionEvents(
              (org.apache.hadoop.mapred.JobID)reduce.getJobID(),
              fromEventIdx,
              maxEventsToFetch,
              (org.apache.hadoop.mapred.TaskAttemptID)reduce);
      events = update.getMapTaskCompletionEvents();
      LOG.debug("Got " + events.length + " map completion events from " +
               fromEventIdx);


      assert !update.shouldReset() : "Unexpected legacy state";


      // Update the last seen event ID
      fromEventIdx += events.length;


      // Process the TaskCompletionEvents:

View Full Code Here

   */  
  private int getMapCompletionEvents() throws IOException {
    
    int numNewMaps = 0;
    
    MapTaskCompletionEventsUpdate update = 
      umbilical.getMapCompletionEvents((org.apache.hadoop.mapred.JobID)
                                       reduce.getJobID(), 
                                       fromEventId, 
                                       MAX_EVENTS_TO_FETCH,
                                       (org.apache.hadoop.mapred.TaskAttemptID)
                                         reduce);
    TaskCompletionEvent events[] = update.getMapTaskCompletionEvents();
    LOG.debug("Got " + events.length + " map completion events from " + 
             fromEventId);
      
    // Check if the reset is required.
    // Since there is no ordering of the task completion events at the 
    // reducer, the only option to sync with the new jobtracker is to reset 
    // the events index
    if (update.shouldReset()) {
      fromEventId = 0;
      scheduler.resetKnownMaps();
    }
    
    // Update the last seen event ID

View Full Code Here

   */  
  private int getMapCompletionEvents() throws IOException {
    
    int numNewMaps = 0;
    
    MapTaskCompletionEventsUpdate update = 
      umbilical.getMapCompletionEvents((org.apache.hadoop.mapred.JobID)
                                       reduce.getJobID(), 
                                       fromEventId, 
                                       MAX_EVENTS_TO_FETCH,
                                       (org.apache.hadoop.mapred.TaskAttemptID)
                                         reduce);
    TaskCompletionEvent events[] = update.getMapTaskCompletionEvents();
    LOG.debug("Got " + events.length + " map completion events from " + 
             fromEventId);
      
    // Check if the reset is required.
    // Since there is no ordering of the task completion events at the 
    // reducer, the only option to sync with the new jobtracker is to reset 
    // the events index
    if (update.shouldReset()) {
      fromEventId = 0;
      scheduler.resetKnownMaps();
    }
    
    // Update the last seen event ID

View Full Code Here

          eventIdx, true, TaskCompletionEvent.Status.SUCCEEDED,
          "http://somehost:8888");
      tceList.add(tce);
    }
    TaskCompletionEvent[] events = {};
    return new MapTaskCompletionEventsUpdate(tceList.toArray(events), false);
  }

View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapred.MapTaskCompletionEventsUpdate

org.apache.hadoop.mapreduce.task.reduce.EventFetcher

org.apache.hadoop.mapreduce.task.reduce.TestEventFetcher

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.