if(flattenedColumns == null || flattenedColumns.size() == 0) {
return false;
}
ProjectionMap foreachProjectionMap = foreach.getProjectionMap();
if(foreachProjectionMap == null) {
return false;
}
List<Integer> foreachAddedFields = foreachProjectionMap.getAddedFields();
if(foreachAddedFields != null) {
Set<Integer> foreachAddedFieldsSet = new HashSet<Integer>(foreachAddedFields);
flattenedColumnSet.removeAll(foreachAddedFieldsSet);
}
if(flattenedColumnSet.size() == 0) {
return false;
}
for(LogicalPlan foreachPlan: foreach.getForEachPlans()) {
UDFFinder udfFinder = new UDFFinder(foreachPlan);
udfFinder.visit();
// if any of the foreach's inner plans contain a UDF then return false
if (udfFinder.foundAnyUDF()) {
return false;
}
CastFinder castFinder = new CastFinder(foreachPlan);
castFinder.visit();
// TODO
// if any of the foreach's inner plans contain a cast then return false
// in the future the cast should be moved appropriately
if (castFinder.foundAnyCast()) {
return false;
}
}
List<LogicalOperator> successors = (mPlan.getSuccessors(foreach) == null ? null
: new ArrayList<LogicalOperator>(mPlan
.getSuccessors(foreach)));
// if the foreach has no successors or more than one successor
// return false
if (successors == null || successors.size() == 0 || successors.size() > 1) {
return false;
}
LogicalOperator successor = successors.get(0);
List<LogicalOperator> peers = (mPlan.getPredecessors(successor) == null ? null
: new ArrayList<LogicalOperator>(mPlan.getPredecessors(successor)));
// check if any of the foreach's peers is a foreach flatten
// if so then this rule does not apply
if (peers != null){
for(LogicalOperator peer: peers) {
if(!peer.equals(foreach)) {
if(peer instanceof LOForEach) {
LOForEach peerForeach = (LOForEach)peer;
if(peerForeach.hasFlatten().first) {
return false;
}
}
}
}
}
IndexHelper<LogicalOperator> indexHelper = new IndexHelper<LogicalOperator>(peers);
Integer foreachPosition = indexHelper.getIndex(foreach);
// Check if flattened fields is required by successor, if so, don't optimize
List<RequiredFields> requiredFieldsList = ((RelationalOperator)successor).getRequiredFields();
RequiredFields requiredFields = requiredFieldsList.get(foreachPosition.intValue());
MultiMap<Integer, Column> foreachMappedFields = foreachProjectionMap.getMappedFields();
if (requiredFields.getFields()!=null) {
for (Pair<Integer, Integer> pair : requiredFields.getFields()) {
Collection<Column> columns = foreachMappedFields.get(pair.second);
if (columns!=null) {
for (Column column : columns) {
Pair<Integer, Integer> foreachInputColumn = column.getInputColumn();
if (foreach.isInputFlattened(foreachInputColumn.second))
return false;
}
}
}
}
// the foreach with flatten can be swapped with an order by
// as the order by will have lesser number of records to sort
// also the sort does not alter the records that are processed
// the foreach with flatten can be pushed down a cross or a join
// for the same reason. In this case the foreach has to be first
// unflattened and then a new foreach has to be inserted after
// the cross or join. In both cross and foreach the actual columns
// from the foreach are not altered but positions might be changed
// in the case of union the column is transformed and as a result
// the foreach flatten cannot be pushed down
// for distinct the output before flattening and the output
// after flattening might be different. For example, consider
// {(1), (1)}. Distinct of this bag is still {(1), (1)}.
// distinct(flatten({(1), (1)})) is (1). However,
// flatten(distinct({(1), (1)})) is (1), (1)
// in both cases correctness is not affected
if(successor instanceof LOSort) {
LOSort sort = (LOSort) successor;
RequiredFields sortRequiredField = sort.getRequiredFields().get(0);
if(sortRequiredField.getNeedAllFields()) {
return false;
}
List<Pair<Integer, Integer>> sortInputs = sortRequiredField.getFields();
Set<Integer> requiredInputs = new HashSet<Integer>();
for(Pair<Integer, Integer> pair: sortInputs) {
requiredInputs.add(pair.second);
}
requiredInputs.retainAll(flattenedColumnSet);
// the intersection of the sort's required inputs
// and the flattened columns in the foreach should
// be null, i.e., the size of required inputs == 0
if(requiredInputs.size() != 0) {
return false;
}
mSwap = true;
return true;
} else if (successor instanceof LOCross
|| successor instanceof LOJoin) {
List<LogicalOperator> children = mPlan.getSuccessors(successor);
if(children == null || children.size() > 1) {
return false;
}
ProjectionMap succProjectionMap = successor.getProjectionMap();
if(succProjectionMap == null) {
return false;
}
MultiMap<Integer, ProjectionMap.Column> mappedFields = succProjectionMap.getMappedFields();
if(mappedFields == null) {
return false;
}