private void processKeyValuePairs(Object key, Object value)
throws HiveException {
String filePath = "";
try {
OrcFileValueWrapper v;
OrcFileKeyWrapper k;
if (key instanceof CombineHiveKey) {
k = (OrcFileKeyWrapper) ((CombineHiveKey) key).getKey();
} else {
k = (OrcFileKeyWrapper) key;
}
filePath = k.getInputPath().toUri().getPath();
fixTmpPath(k.getInputPath().getParent());
v = (OrcFileValueWrapper) value;
if (prevPath == null) {
prevPath = k.getInputPath();
reader = OrcFile.createReader(fs, k.getInputPath());
LOG.info("ORC merge file input path: " + k.getInputPath());
}
// store the orc configuration from the first file. All other files should
// match this configuration before merging else will not be merged
if (outWriter == null) {
compression = k.getCompression();
compressBuffSize = k.getCompressBufferSize();
version = k.getVersionList();
columnCount = k.getTypes().get(0).getSubtypesCount();
rowIndexStride = k.getRowIndexStride();
// block size and stripe size will be from config
outWriter = OrcFile.createWriter(outPath,
OrcFile.writerOptions(jc).compress(compression)
.inspector(reader.getObjectInspector()));
LOG.info("ORC merge file output path: " + outPath);
}
if (!checkCompatibility(k)) {
incompatFileSet.add(k.getInputPath());
return;
}
// next file in the path
if (!k.getInputPath().equals(prevPath)) {
reader = OrcFile.createReader(fs, k.getInputPath());
}
// initialize buffer to read the entire stripe
byte[] buffer = new byte[(int) v.getStripeInformation().getLength()];
fdis = fs.open(k.getInputPath());
fdis.readFully(v.getStripeInformation().getOffset(), buffer, 0,
(int) v.getStripeInformation().getLength());
// append the stripe buffer to the new ORC file
outWriter.appendStripe(buffer, 0, buffer.length, v.getStripeInformation(),
v.getStripeStatistics());
LOG.info("Merged stripe from file " + k.getInputPath() + " [ offset : "
+ v.getStripeInformation().getOffset() + " length: "
+ v.getStripeInformation().getLength() + " ]");
// add user metadata to footer in case of any
if (v.isLastStripeInFile()) {