ArrayList<ArrayList<OperatorKey>> inpTargets =
(ArrayList<ArrayList<OperatorKey>>)
ObjectSerializer.deserialize(jobConf.get("pig.inpTargets"));
intermediateData.clear();
MapReduceOper mro = jobToMroMap.get(job);
PigSplit split = null;
List<POStore> stores = null;
PhysicalOperator pack = null;
// revisit as there are new physical operators from MR compilation
if (!mro.mapPlan.isEmpty())
attacher.revisit(mro.mapPlan);
if (!mro.reducePlan.isEmpty()) {
attacher.revisit(mro.reducePlan);
pack = mro.reducePlan.getRoots().get(0);
}
List<POLoad> lds = PlanHelper.getLoads(mro.mapPlan);
if (!mro.mapPlan.isEmpty()) {
stores = PlanHelper.getStores(mro.mapPlan);
}
if (!mro.reducePlan.isEmpty()) {
if (stores == null)
stores = PlanHelper.getStores(mro.reducePlan);
else
stores.addAll(PlanHelper.getStores(mro.reducePlan));
}
for (POStore store : stores) {
output.put(store.getSFile().getFileName(), attacher.getDataMap().get(store));
}
OutputAttacher oa = new OutputAttacher(mro.mapPlan, output);
oa.visit();
if (!mro.reducePlan.isEmpty()) {
oa = new OutputAttacher(mro.reducePlan, output);
oa.visit();
}
int index = 0;
for (POLoad ld : lds) {
input = output.get(ld.getLFile().getFileName());
if (input == null && baseData != null) {
for (LogicalRelationalOperator lo : baseData.keySet()) {
if (((LOLoad) lo).getSchemaFile().equals(ld.getLFile().getFileName()))
{
input = baseData.get(lo);
break;
}
}
}
if (input != null)
mro.mapPlan.remove(ld);
}
for (POLoad ld : lds) {
// check newly generated data first
input = output.get(ld.getLFile().getFileName());
if (input == null && baseData != null) {
if (input == null && baseData != null) {
for (LogicalRelationalOperator lo : baseData.keySet()) {
if (((LOLoad) lo).getSchemaFile().equals(ld.getLFile().getFileName()))
{
input = baseData.get(lo);
break;
}
}
}
}
needFileInput = (input == null);
split = new PigSplit(null, index, needFileInput ? emptyInpTargets : inpTargets.get(index), 0);
++index;
Mapper<Text, Tuple, PigNullableWritable, Writable> map;
if (mro.reducePlan.isEmpty()) {
// map-only