@Override
public FileInputSplit[] createInputSplits(int minNumSplits) throws IOException {
List<FileStatus> files = this.getFiles();
final FileSystem fs = this.filePath.getFileSystem();
final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? fs.getDefaultBlockSize() : this.blockSize;
final List<FileInputSplit> inputSplits = new ArrayList<FileInputSplit>(minNumSplits);
for (FileStatus file : files) {
long splitSize = blockSize;
for (long pos = 0, length = file.getLen(); pos < length; pos += splitSize) {
long remainingLength = Math.min(pos + splitSize, length) - pos;
// get the block locations and make sure they are in order with respect to their offset
final BlockLocation[] blocks = fs.getFileBlockLocations(file, pos, remainingLength);
Arrays.sort(blocks);
inputSplits.add(new FileInputSplit(inputSplits.size(), file.getPath(), pos, remainingLength,
blocks[0].getHosts()));
}
}
if (inputSplits.size() < minNumSplits) {
LOG.warn(String.format(
"With the given block size %d, the file %s cannot be split into %d blocks. Filling up with empty splits...",
blockSize, this.filePath, minNumSplits));
FileStatus last = files.get(files.size() - 1);
final BlockLocation[] blocks = fs.getFileBlockLocations(last, 0, last.getLen());
for (int index = files.size(); index < minNumSplits; index++) {
inputSplits.add(new FileInputSplit(index, last.getPath(), last.getLen(), 0, blocks[0].getHosts()));
}
}