? kvend
: kvoffsets.length + kvend;
//record the cumulative resources used before running sort
long sortStartMilli = System.currentTimeMillis();
ProcResourceValues sortStartProcVals = getCurrentProcResourceValues();
//do the sort
sorter.sort(MapOutputBuffer.this, kvstart, endPosition, reporter);
// get the cumulative resources used after the sort, and use the diff as
// resources/wallclock consumed by the sort.
long sortEndMilli = System.currentTimeMillis();
ProcResourceValues sortEndProcVals = getCurrentProcResourceValues();
spillSortCounters.incCountersPerSort(sortStartProcVals,
sortEndProcVals, sortEndMilli - sortStartMilli);
int spindex = kvstart;
IndexRecord rec = new IndexRecord();
InMemValBytes value = new InMemValBytes();
long spillBytes = 0;
for (int i = 0; i < partitions; ++i) {
IFile.Writer<K, V> writer = null;
try {
long segmentStart = out.getPos();
writer = new Writer<K, V>(job, out, keyClass, valClass, codec,
spilledRecordsCounter);
if (combinerRunner == null) {
// spill directly
DataInputBuffer key = new DataInputBuffer();
while (spindex < endPosition &&
kvindices[kvoffsets[spindex % kvoffsets.length]
+ PARTITION] == i) {
final int kvoff = kvoffsets[spindex % kvoffsets.length];
getVBytesForOffset(kvoff, value);
key.reset(kvbuffer, kvindices[kvoff + KEYSTART],
(kvindices[kvoff + VALSTART] -
kvindices[kvoff + KEYSTART]));
writer.append(key, value);
++spindex;
}
} else {
int spstart = spindex;
while (spindex < endPosition &&
kvindices[kvoffsets[spindex % kvoffsets.length]
+ PARTITION] == i) {
++spindex;
}
// Note: we would like to avoid the combiner if we've fewer
// than some threshold of records for a partition
if (spstart != spindex) {
combineCollector.setWriter(writer);
RawKeyValueIterator kvIter =
new MRResultIterator(spstart, spindex);
combinerRunner.combine(kvIter, combineCollector);
}
}
// close the writer
writer.close();
// record offsets
rec.startOffset = segmentStart;
rec.rawLength = writer.getRawLength();
rec.partLength = writer.getCompressedLength();
spillBytes += writer.getCompressedLength();
spillRec.putIndex(rec, i);
writer = null;
} finally {
if (null != writer) writer.close();
}
}
if (totalIndexCacheMemory >= INDEX_CACHE_MEMORY_LIMIT) {
// create spill index file
Path indexFilename = mapOutputFile.getSpillIndexFileForWrite(
getTaskID(), numSpills,
partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH);
spillRec.writeToFile(indexFilename, job);
} else {
indexCacheList.add(spillRec);
totalIndexCacheMemory +=
spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH;
}
long spillEndMilli = System.currentTimeMillis();
ProcResourceValues spillEndProcVals = getCurrentProcResourceValues();
spillSortCounters.incCountersPerSpill(sortEndProcVals,
spillEndProcVals, spillEndMilli - sortEndMilli, spillBytes);
LOG.info("Finished spill " + numSpills);
++numSpills;