totalBytes += it.next().getSize();
}
long goalSize = totalBytes / (numSplits < 1 ? 1 : numSplits);
StringBuilder sb = new StringBuilder();
boolean first = true;
PathFilter filter = null;
List<BasicTable.Reader> realReaders = new ArrayList<BasicTable.Reader>();
int[] realReaderIndices = new int[readers.size()];
for (int i = 0; i < readers.size(); ++i) {
BasicTable.Reader reader = readers.get(i);
/* Get the index of the column group that will be used for row-split.*/
int splitCGIndex = reader.getRowSplitCGIndex();
/* We can create input splits only if there does exist a valid column group for split.
* Otherwise, we do not create input splits. */
if (splitCGIndex >= 0) {
realReaderIndices[realReaders.size()] = i;
realReaders.add(reader);
if (first)
{
// filter is identical across tables
filter = reader.getPathFilter(conf);
first = false;
} else
sb.append(",");
sb.append(reader.getPath().toString() + "/" + reader.getName(splitCGIndex));
}
}
DummyFileInputFormat helper = new DummyFileInputFormat(minSplitSize, realReaders);
if (!realReaders.isEmpty())
{
DummyFileInputFormat.setInputPaths(conf, sb.toString());
DummyFileInputFormat.setInputPathFilter(conf, filter.getClass());
InputSplit[] inputSplits = helper.getSplits(conf, (numSplits < 1 ? 1 : numSplits));
int batchesPerSplit = inputSplits.length / (numSplits < 1 ? 1 : numSplits);
if (batchesPerSplit <= 0)
batchesPerSplit = 1;