Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.TaskCompletionEvent

150
151
152
153
154
155
156
157
158
159
160
161
162
    Assert.assertEquals("Event status not correct for map attempt2",
        TaskAttemptCompletionEventStatus.SUCCEEDED, events[2].getStatus());
    Assert.assertEquals("Event status not correct for reduce attempt1",
        TaskAttemptCompletionEventStatus.SUCCEEDED, events[3].getStatus());

    TaskCompletionEvent mapEvents[] =
        job.getMapAttemptCompletionEvents(0, 2);
    TaskCompletionEvent convertedEvents[] = TypeConverter.fromYarn(events);
    Assert.assertEquals("Incorrect number of map events", 2, mapEvents.length);
    Assert.assertArrayEquals("Unexpected map events",
        Arrays.copyOfRange(convertedEvents, 0, 2), mapEvents);
    mapEvents = job.getMapAttemptCompletionEvents(2, 200);
    Assert.assertEquals("Incorrect number of map events", 1, mapEvents.length);
View Full Code Here
397
398
399
400
401
402
403
404
405
406
407
408
409
    Assert.assertEquals("Event status not correct for map attempt2",
        TaskAttemptCompletionEventStatus.SUCCEEDED, events[2].getStatus());
    Assert.assertEquals("Event status not correct for reduce attempt1",
        TaskAttemptCompletionEventStatus.SUCCEEDED, events[3].getStatus());

    TaskCompletionEvent mapEvents[] =
        job.getMapAttemptCompletionEvents(0, 2);
    TaskCompletionEvent convertedEvents[] = TypeConverter.fromYarn(events);
    Assert.assertEquals("Incorrect number of map events", 2, mapEvents.length);
    Assert.assertArrayEquals("Unexpected map events",
        Arrays.copyOfRange(convertedEvents, 0, 2), mapEvents);
    mapEvents = job.getMapAttemptCompletionEvents(2, 200);
    Assert.assertEquals("Incorrect number of map events", 1, mapEvents.length);
View Full Code Here
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
        successEvent.setStatus(TaskAttemptCompletionEventStatus.OBSOLETE);
        int mapCompletionIdx =
            job.taskCompletionIdxToMapCompletionIdx.get(successEventNo);
        if (mapCompletionIdx >= 0) {
          // update the corresponding TaskCompletionEvent for the map
          TaskCompletionEvent mapEvent =
              job.mapAttemptCompletionEvents.get(mapCompletionIdx);
          job.mapAttemptCompletionEvents.set(mapCompletionIdx,
              new TaskCompletionEvent(mapEvent.getEventId(),
                  mapEvent.getTaskAttemptId(), mapEvent.idWithinJob(),
                  mapEvent.isMapTask(), TaskCompletionEvent.Status.OBSOLETE,
                  mapEvent.getTaskTrackerHttp()));
        }
      }
     
      // if this attempt is not successful then why is the previous successful
      // attempt being removed above - MAPREDUCE-4330
View Full Code Here
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
        successEvent.setStatus(TaskAttemptCompletionEventStatus.OBSOLETE);
        int mapCompletionIdx =
            job.taskCompletionIdxToMapCompletionIdx.get(successEventNo);
        if (mapCompletionIdx >= 0) {
          // update the corresponding TaskCompletionEvent for the map
          TaskCompletionEvent mapEvent =
              job.mapAttemptCompletionEvents.get(mapCompletionIdx);
          job.mapAttemptCompletionEvents.set(mapCompletionIdx,
              new TaskCompletionEvent(mapEvent.getEventId(),
                  mapEvent.getTaskAttemptId(), mapEvent.idWithinJob(),
                  mapEvent.isMapTask(), TaskCompletionEvent.Status.OBSOLETE,
                  mapEvent.getTaskTrackerHttp()));
        }
      }
     
      // if this attempt is not successful then why is the previous successful
      // attempt being removed above - MAPREDUCE-4330
View Full Code Here
168
169
170
171
172
173
174
175
176
177
178
    return oldEvents;
  }

  public static TaskCompletionEvent fromYarn(
      TaskAttemptCompletionEvent newEvent) {
    return new TaskCompletionEvent(newEvent.getEventId(),
              fromYarn(newEvent.getAttemptId()), newEvent.getAttemptId().getId(),
              newEvent.getAttemptId().getTaskId().getTaskType().equals(TaskType.MAP),
              fromYarn(newEvent.getStatus()),
              newEvent.getMapOutputServerAddress());
  }
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
                                       reduce.getJobID(),
                                       fromEventId,
                                       MAX_EVENTS_TO_FETCH,
                                       (org.apache.hadoop.mapred.TaskAttemptID)
                                         reduce);
    TaskCompletionEvent events[] = update.getMapTaskCompletionEvents();
    LOG.debug("Got " + events.length + " map completion events from " +
             fromEventId);
     
    // Check if the reset is required.
    // Since there is no ordering of the task completion events at the
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
                                       reduce.getJobID(),
                                       fromEventId,
                                       MAX_EVENTS_TO_FETCH,
                                       (org.apache.hadoop.mapred.TaskAttemptID)
                                         reduce);
    TaskCompletionEvent events[] = update.getMapTaskCompletionEvents();
    LOG.debug("Got " + events.length + " map completion events from " +
             fromEventId);
     
    // Check if the reset is required.
    // Since there is no ordering of the task completion events at the
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
      int startIdx, int numEvents) {
    ArrayList<TaskCompletionEvent> tceList =
        new ArrayList<TaskCompletionEvent>(numEvents);
    for (int i = 0; i < numEvents; ++i) {
      int eventIdx = startIdx + i;
      TaskCompletionEvent tce = new TaskCompletionEvent(eventIdx,
          new TaskAttemptID("12345", 1, TaskType.MAP, eventIdx, 0),
          eventIdx, true, TaskCompletionEvent.Status.SUCCEEDED,
          "http://somehost:8888");
      tceList.add(tce);
    }
View Full Code Here
180
181
182
183
184
185
186
187
188
189
190
    return oldEvents;
  }

  public static TaskCompletionEvent fromYarn(
      TaskAttemptCompletionEvent newEvent) {
    return new TaskCompletionEvent(newEvent.getEventId(),
              fromYarn(newEvent.getAttemptId()), newEvent.getAttemptId().getId(),
              newEvent.getAttemptId().getTaskId().getTaskType().equals(TaskType.MAP),
              fromYarn(newEvent.getStatus()),
              newEvent.getMapOutputServerAddress());
  }
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
161
162
    Assert.assertEquals("Event status not correct for map attempt2",
        TaskAttemptCompletionEventStatus.SUCCEEDED, events[2].getStatus());
    Assert.assertEquals("Event status not correct for reduce attempt1",
        TaskAttemptCompletionEventStatus.SUCCEEDED, events[3].getStatus());

    TaskCompletionEvent mapEvents[] =
        job.getMapAttemptCompletionEvents(0, 2);
    TaskCompletionEvent convertedEvents[] = TypeConverter.fromYarn(events);
    Assert.assertEquals("Incorrect number of map events", 2, mapEvents.length);
    Assert.assertArrayEquals("Unexpected map events",
        Arrays.copyOfRange(convertedEvents, 0, 2), mapEvents);
    mapEvents = job.getMapAttemptCompletionEvents(2, 200);
    Assert.assertEquals("Incorrect number of map events", 1, mapEvents.length);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.TaskCompletionEvent

  • org.apache.hadoop.mapreduce.task.reduce.EventFetcher
  • org.apache.hadoop.mapreduce.task.reduce.TestEventFetcher
  • org.apache.hadoop.mapreduce.TypeConverter
  • org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl$TaskAttemptCompletedEventTransition
  • org.apache.hadoop.mapreduce.v2.app.TestFetchFailure
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.