ReduceSinkOperator rsOp = (ReduceSinkOperator)OperatorFactory.getAndMakeChild(rsDesc, fsRS, ts_op);
mapredWork cplan = GenMapRedUtils.getMapRedWork();
ParseContext parseCtx = ctx.getParseCtx();
Task<? extends Serializable> mergeTask = TaskFactory.get(cplan, parseCtx.getConf());
fileSinkDesc fsConf = fsOp.getConf();
// Add the extract operator to get the value fields
RowResolver out_rwsch = new RowResolver();
RowResolver interim_rwsch = ctx.getParseCtx().getOpParseCtx().get(fsOp).getRR();
Integer pos = Integer.valueOf(0);
for(ColumnInfo colInfo: interim_rwsch.getColumnInfos()) {
String [] info = interim_rwsch.reverseLookup(colInfo.getInternalName());
out_rwsch.put(info[0], info[1],
new ColumnInfo(pos.toString(), colInfo.getType(), info[0],
colInfo.getIsPartitionCol()));
pos = Integer.valueOf(pos.intValue() + 1);
}
Operator extract =
OperatorFactory.getAndMakeChild(
new extractDesc(new exprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,
Utilities.ReduceField.VALUE.toString(), "", false)),
new RowSchema(out_rwsch.getColumnInfos()));
tableDesc ts = (tableDesc)fsConf.getTableInfo().clone();
fsConf.getTableInfo().getProperties().remove(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_PARTITION_COLUMNS);
FileSinkOperator newOutput =
(FileSinkOperator)OperatorFactory.getAndMakeChild(
new fileSinkDesc(finalName, ts,
parseCtx.getConf().getBoolVar(HiveConf.ConfVars.COMPRESSRESULT)),
fsRS, extract);
cplan.setReducer(extract);
ArrayList<String> aliases = new ArrayList<String>();
aliases.add(fsConf.getDirName());
cplan.getPathToAliases().put(fsConf.getDirName(), aliases);
cplan.getAliasToWork().put(fsConf.getDirName(), ts_op);
cplan.getPathToPartitionInfo().put(fsConf.getDirName(), new partitionDesc(fsConf.getTableInfo(), null));
cplan.setNumReduceTasks(-1);
moveWork dummyMv = new moveWork(null, null, null, new loadFileDesc(fsOp.getConf().getDirName(), finalName, true, null, null), false);
Task<? extends Serializable> dummyMergeTask = TaskFactory.get(dummyMv, ctx.getConf());
List<Serializable> listWorks = new ArrayList<Serializable>();