Iterator<InputAttemptIdentifier> listItr = origList.iterator();
while (listItr.hasNext()) {
// we may want to try all versions of the input but with current retry
// behavior older ones are likely to be lost and should be ignored.
// This may be removed after TEZ-914
InputAttemptIdentifier id = listItr.next();
if (inputShouldBeConsumed(id)) {
Integer inputNumber = new Integer(id.getInputIdentifier().getInputIndex());
InputAttemptIdentifier oldId = dedupedList.get(inputNumber);
if (oldId == null || oldId.getAttemptNumber() < id.getAttemptNumber()) {
dedupedList.put(inputNumber, id);
if (oldId != null) {
LOG.warn("Old Src for InputIndex: " + inputNumber + " with attemptNumber: "
+ oldId.getAttemptNumber()
+ " was not determined to be invalid. Ignoring it for now in favour of "
+ id.getAttemptNumber());
}
}
} else {
LOG.info("Ignoring finished or obsolete source: " + id);
}
}
// Compute the final list, limited by NUM_FETCHERS_AT_ONCE
List<InputAttemptIdentifier> result = new ArrayList<InputAttemptIdentifier>();
int includedMaps = 0;
int totalSize = dedupedList.size();
Iterator<Map.Entry<Integer, InputAttemptIdentifier>> dedupedItr = dedupedList.entrySet().iterator();
// find the maps that we still need, up to the limit
while (dedupedItr.hasNext()) {
InputAttemptIdentifier id = dedupedItr.next().getValue();
result.add(id);
if (++includedMaps >= maxTaskOutputAtOnce) {
break;
}
}
// put back the maps left after the limit
while (dedupedItr.hasNext()) {
InputAttemptIdentifier id = dedupedItr.next().getValue();
host.addKnownMap(id);
}
LOG.info("assigned " + includedMaps + " of " + totalSize + " to " +
host + " to " + Thread.currentThread().getName());
return result;