* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

    private static final Log LOG = LogFactory.getLog(LogToPhyTranslationVisitor.class);

    public LogToPhyTranslationVisitor(OperatorPlan plan) throws FrontendException {

        super(plan, new DependencyOrderWalker(plan));

        currentPlan = new PhysicalPlan();

        logToPhyMap = new HashMap<Operator, PhysicalOperator>();

        currentPlans = new LinkedList<PhysicalPlan>();

    }

    protected Map<Operator, PhysicalOperator> logToPhyMap;

    protected Deque<PhysicalPlan> currentPlans;

    protected PhysicalPlan currentPlan;

    protected NodeIdGenerator nodeGen = NodeIdGenerator.getGenerator();

    protected PigContext pc;

    public void setPigContext(PigContext pc) {

        this.pc = pc;

    }

    public Map<Operator, PhysicalOperator> getLogToPhyMap() {

        return logToPhyMap;

    }

    public PhysicalPlan getPhysicalPlan() {

        return currentPlan;

    }

    @Override

    public void visit(LOLoad loLoad) throws FrontendException {

        String scope = DEFAULT_SCOPE;

        // The last parameter here is set to true as we assume all files are

        // splittable due to LoadStore Refactor

        POLoad load = new POLoad(new OperatorKey(scope, nodeGen

                .getNextNodeId(scope)), loLoad.getLoadFunc());

        load.addOriginalLocation(loLoad.getAlias(), loLoad.getLocation());

        load.setLFile(loLoad.getFileSpec());

        load.setPc(pc);

        load.setResultType(DataType.BAG);

        load.setSignature(loLoad.getSignature());

        load.setLimit(loLoad.getLimit());

        load.setIsTmpLoad(loLoad.isTmpLoad());

        load.setCacheFiles(loLoad.getLoadFunc().getCacheFiles());

        load.setShipFiles(loLoad.getLoadFunc().getShipFiles());

        currentPlan.add(load);

        logToPhyMap.put(loLoad, load);

        // Load is typically a root operator, but in the multiquery

        // case it might have a store as a predecessor.

        List<Operator> op = loLoad.getPlan().getPredecessors(loLoad);

        PhysicalOperator from;

        if(op != null) {

            from = logToPhyMap.get(op.get(0));

            try {

                currentPlan.connect(from, load);

            } catch (PlanException e) {

                int errCode = 2015;

                String msg = "Invalid physical operators in the physical plan" ;

                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);

            }

        }

    }

    @Override

    public void visit(LONative loNative) throws FrontendException{

        String scope = DEFAULT_SCOPE;

        PONative poNative = new PONative(new OperatorKey(scope, nodeGen

                .getNextNodeId(scope)));

        poNative.addOriginalLocation(loNative.getAlias(), loNative.getLocation());

        poNative.setNativeMRjar(loNative.getNativeMRJar());

        poNative.setParams(loNative.getParams());

        poNative.setResultType(DataType.BAG);

        logToPhyMap.put(loNative, poNative);

        currentPlan.add(poNative);

        List<Operator> op = loNative.getPlan().getPredecessors(loNative);

        PhysicalOperator from;

        if(op != null) {

            from = logToPhyMap.get(op.get(0));

        } else {

            int errCode = 2051;

            String msg = "Did not find a predecessor for Native." ;

            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG);

        }

        try {

            currentPlan.connect(from, poNative);

        } catch (PlanException e) {

            int errCode = 2015;

            String msg = "Invalid physical operators in the physical plan" ;

            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);

        }

    }

    @Override

    public void visit(LOFilter filter) throws FrontendException {

        String scope = DEFAULT_SCOPE;

        //        System.err.println("Entering Filter");

        POFilter poFilter = new POFilter(new OperatorKey(scope, nodeGen

                .getNextNodeId(scope)), filter.getRequestedParallelism());

        poFilter.addOriginalLocation(filter.getAlias(), filter.getLocation());

        poFilter.setResultType(DataType.BAG);

        currentPlan.add(poFilter);

        logToPhyMap.put(filter, poFilter);

        currentPlans.push(currentPlan);

        currentPlan = new PhysicalPlan();

        //        PlanWalker childWalker = currentWalker

        //                .spawnChildWalker(filter.getFilterPlan());

        PlanWalker childWalker = new ReverseDependencyOrderWalkerWOSeenChk(filter.getFilterPlan());

        pushWalker(childWalker);

        //currentWalker.walk(this);

        currentWalker.walk(

                new ExpToPhyTranslationVisitor( currentWalker.getPlan(),

                        childWalker, filter, currentPlan, logToPhyMap ) );

        popWalker();

        poFilter.setPlan(currentPlan);

        currentPlan = currentPlans.pop();

        List<Operator> op = filter.getPlan().getPredecessors(filter);

        PhysicalOperator from;

        if(op != null) {

            from = logToPhyMap.get(op.get(0));

        } else {

            int errCode = 2051;

            String msg = "Did not find a predecessor for Filter." ;

            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG);

        }

        try {

            currentPlan.connect(from, poFilter);

        } catch (PlanException e) {

            int errCode = 2015;

            String msg = "Invalid physical operators in the physical plan" ;

            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);

        }

        translateSoftLinks(filter);

        //        System.err.println("Exiting Filter");

    }

    @Override

    public void visit(LOSort sort) throws FrontendException {

        String scope = DEFAULT_SCOPE;

        List<LogicalExpressionPlan> logPlans = sort.getSortColPlans();

        List<PhysicalPlan> sortPlans = new ArrayList<PhysicalPlan>(logPlans.size());

        // convert all the logical expression plans to physical expression plans

        currentPlans.push(currentPlan);

        for (LogicalExpressionPlan plan : logPlans) {

            currentPlan = new PhysicalPlan();

            PlanWalker childWalker = new ReverseDependencyOrderWalkerWOSeenChk(plan);

            pushWalker(childWalker);

            childWalker.walk(new ExpToPhyTranslationVisitor( currentWalker.getPlan(),

                    childWalker, sort, currentPlan, logToPhyMap));

            sortPlans.add(currentPlan);

            popWalker();

        }

        currentPlan = currentPlans.pop();

        // get the physical operator for sort

        POSort poSort;

        if (sort.getUserFunc() == null) {

            poSort = new POSort(new OperatorKey(scope, nodeGen

                    .getNextNodeId(scope)), sort.getRequestedParallelism(), null,

                    sortPlans, sort.getAscendingCols(), null);

        } else {

            POUserComparisonFunc comparator = new POUserComparisonFunc(new OperatorKey(

                    scope, nodeGen.getNextNodeId(scope)), sort

                    .getRequestedParallelism(), null, sort.getUserFunc());

            poSort = new POSort(new OperatorKey(scope, nodeGen

                    .getNextNodeId(scope)), sort.getRequestedParallelism(), null,

                    sortPlans, sort.getAscendingCols(), comparator);

        }

        poSort.addOriginalLocation(sort.getAlias(), sort.getLocation());

        poSort.setLimit(sort.getLimit());

        // sort.setRequestedParallelism(s.getType());

        logToPhyMap.put(sort, poSort);

        currentPlan.add(poSort);

        List<Operator> op = sort.getPlan().getPredecessors(sort);

        PhysicalOperator from;

        if(op != null) {

            from = logToPhyMap.get(op.get(0));

        } else {

            int errCode = 2051;

            String msg = "Did not find a predecessor for Sort." ;

            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG);

        }

        try {

            currentPlan.connect(from, poSort);

        } catch (PlanException e) {

            int errCode = 2015;

            String msg = "Invalid physical operators in the physical plan" ;

            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);

        }

        poSort.setResultType(DataType.BAG);

    }

    /**

     * Transformation from Logical to Physical Plan involves the following steps:

     * First, it is generated a random number which will link a POCounter within a PORank.

     * On this way, avoiding possible collisions on parallel rank operations.

     * Then, if it is row number mode:

     * <pre>

     * In case of a RANK operation (row number mode), are used two steps:

     *   1.- Each tuple is counted sequentially on each mapper, and are produced global counters

     *   2.- Global counters are gathered and summed, each tuple calls to the respective counter value

     *       in order to calculate the corresponding rank value.

     * </pre>

     * or not:

     * <pre>

     * In case of a RANK BY operation, then are necessary five steps:

     *   1.- Group by the fields involved on the rank operation: POPackage

     *   2.- In case of multi-fields, the key (group field) is flatten: POForEach

     *   3.- Sort operation by the fields available after flattening: POSort

     *   4.- Each group is sequentially counted on each mapper through a global counter: POCounter

     *   5.- Global counters are summed and passed to the rank operation: PORank

     * </pre>

     * @param loRank describe if the rank operation is on a row number mode

     * or is rank by (dense or not)

     **/

    @Override

    public void visit(LORank loRank) throws FrontendException {

        String scope = DEFAULT_SCOPE;

        PORank poRank;

        POCounter poCounter;

        Random randomGenerator = new Random();

        Long operationID = Math.abs(randomGenerator.nextLong());

        try {

            // Physical operations for RANK operator:

            // In case of a RANK BY operation, then are necessary five steps:

            //   1.- Group by the fields involved on the rank operation: POPackage

            //   2.- In case of multi-fields, the key (group field) is flatten: POForEach

            //   3.- Sort operation by the fields available after flattening: POSort

            //   4.- Each group is sequentially counted on each mapper through a global counter: POCounter

            //   5.- Global counters are summed and passed to the rank operation: PORank

            if(!loRank.isRowNumber()) {

                boolean[] flags = {false};

                MultiMap<Integer, LogicalExpressionPlan> expressionPlans = new MultiMap<Integer, LogicalExpressionPlan>();

                for(int i = 0 ; i < loRank.getRankColPlans().size() ; i++)

                    expressionPlans.put(i,loRank.getRankColPlans());

                POPackage poPackage = compileToLR_GR_PackTrio(loRank, null, flags, expressionPlans);

                poPackage.getPkgr().setPackageType(PackageType.GROUP);

                translateSoftLinks(loRank);

                List<Boolean> flattenLst = Arrays.asList(true, false);

                PhysicalPlan fep1 = new PhysicalPlan();

                POProject feproj1 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), -1);

                feproj1.addOriginalLocation(loRank.getAlias(), loRank.getLocation());

                feproj1.setColumn(0);

                feproj1.setResultType(poPackage.getPkgr().getKeyType());

                feproj1.setStar(false);

                feproj1.setOverloaded(false);

                fep1.add(feproj1);

                PhysicalPlan fep2 = new PhysicalPlan();

                POProject feproj2 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), -1);

                feproj2.addOriginalLocation(loRank.getAlias(), loRank.getLocation());

                feproj2.setColumn(1);

                feproj2.setResultType(DataType.BAG);

                feproj2.setStar(false);

                feproj2.setOverloaded(false);

                fep2.add(feproj2);

                List<PhysicalPlan> fePlans = Arrays.asList(fep1, fep2);

                POForEach poForEach = new POForEach(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), -1, fePlans, flattenLst);

                List<LogicalExpressionPlan> rankPlans = loRank.getRankColPlans();

                byte[] newTypes = new byte[rankPlans.size()];

                for(int i = 0; i < rankPlans.size(); i++) {

                    LogicalExpressionPlan loep = rankPlans.get(i);

                    Iterator<Operator> inpOpers = loep.getOperators();

                    while(inpOpers.hasNext()) {

                        Operator oper = inpOpers.next();

                        newTypes[i] = ((ProjectExpression) oper).getType();

                    }

                }

                List<PhysicalPlan> newPhysicalPlan = new ArrayList<PhysicalPlan>();

                List<Boolean> newOrderPlan = new ArrayList<Boolean>();

                for(int i = 0; i < loRank.getRankColPlans().size(); i++) {

                    PhysicalPlan fep3 = new PhysicalPlan();

                    POProject feproj3 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), -1);

                    feproj3.addOriginalLocation(loRank.getAlias(), loRank.getLocation());

                    feproj3.setColumn(i);

                    feproj3.setResultType(newTypes[i]);

                    feproj3.setStar(false);

                    feproj3.setOverloaded(false);

                    fep3.add(feproj3);

                    newPhysicalPlan.add(fep3);

                    newOrderPlan.add(loRank.getAscendingCol().get(i));

                }

                POSort poSort;

                poSort = new POSort(new OperatorKey(scope, nodeGen

                        .getNextNodeId(scope)), -1, null,

                        newPhysicalPlan, newOrderPlan, null);

                poSort.addOriginalLocation(loRank.getAlias(), loRank.getLocation());

                poCounter = new POCounter(

                        new OperatorKey(scope, nodeGen

                                .getNextNodeId(scope)), -1 , null,

                                newPhysicalPlan, newOrderPlan);

                poCounter.addOriginalLocation(loRank.getAlias(), loRank.getLocation());

                poCounter.setResultType(DataType.TUPLE);

                poCounter.setIsRowNumber(loRank.isRowNumber());

                poCounter.setIsDenseRank(loRank.isDenseRank());

                poCounter.setOperationID(String.valueOf(operationID));

                poRank = new PORank(

                        new OperatorKey(scope, nodeGen

                                .getNextNodeId(scope)), -1 , null,

                                newPhysicalPlan, newOrderPlan);

                poRank.addOriginalLocation(loRank.getAlias(), loRank.getLocation());

                poRank.setResultType(DataType.TUPLE);

                poRank.setOperationID(String.valueOf(operationID));

                List<Boolean> flattenLst2 = Arrays.asList(false, true);

                PhysicalPlan fep12 = new PhysicalPlan();

                POProject feproj12 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), -1);

                feproj12.addOriginalLocation(loRank.getAlias(), loRank.getLocation());

                feproj12.setColumn(0);

                feproj12.setResultType(DataType.LONG);

                feproj12.setStar(false);

                feproj12.setOverloaded(false);

                fep12.add(feproj12);

                PhysicalPlan fep22 = new PhysicalPlan();

                POProject feproj22 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), -1);

                feproj22.addOriginalLocation(loRank.getAlias(), loRank.getLocation());

                feproj22.setColumn(loRank.getRankColPlans().size()+1);

                feproj22.setResultType(DataType.BAG);

                feproj22.setStar(false);

                feproj22.setOverloaded(false);

                fep22.add(feproj22);

                List<PhysicalPlan> fePlans2 = Arrays.asList(fep12, fep22);

                POForEach poForEach2 = new POForEach(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), -1, fePlans2, flattenLst2);

                currentPlan.add(poForEach);

                currentPlan.add(poSort);

                currentPlan.add(poCounter);

                currentPlan.add(poRank);

                currentPlan.add(poForEach2);

                try {

                    currentPlan.connect(poPackage, poForEach);

                    currentPlan.connect(poForEach, poSort);

                    currentPlan.connect(poSort, poCounter);

                    currentPlan.connect(poCounter, poRank);

                    currentPlan.connect(poRank, poForEach2);

                } catch (PlanException e) {

                    throw new LogicalToPhysicalTranslatorException(e.getMessage(),e.getErrorCode(),e.getErrorSource(),e);

                }

                logToPhyMap.put(loRank, poForEach2);

                // In case of a RANK operation, are used two steps:

                //   1.- Each tuple is counted sequentially on each mapper, and are produced global counters

                //   2.- Global counters are gathered and summed, each tuple calls to the respective counter value

                //       in order to calculate the corresponding rank value.

            } else {

                List<LogicalExpressionPlan> logPlans = loRank.getRankColPlans();

                List<PhysicalPlan> rankPlans = new ArrayList<PhysicalPlan>(logPlans.size());

                // convert all the logical expression plans to physical expression plans

                currentPlans.push(currentPlan);

                for (LogicalExpressionPlan plan : logPlans) {

                    currentPlan = new PhysicalPlan();

                    PlanWalker childWalker = new ReverseDependencyOrderWalkerWOSeenChk(plan);

                    pushWalker(childWalker);

                    childWalker.walk(new ExpToPhyTranslationVisitor( currentWalker.getPlan(),

                            childWalker, loRank, currentPlan, logToPhyMap));

                    rankPlans.add(currentPlan);

                    popWalker();

                }

                currentPlan = currentPlans.pop();

                poCounter = new POCounter(

                        new OperatorKey(scope, nodeGen

                                .getNextNodeId(scope)), -1 , null,

                                rankPlans, loRank.getAscendingCol());

                poCounter.addOriginalLocation(loRank.getAlias(), loRank.getLocation());

                poCounter.setResultType(DataType.TUPLE);

                poCounter.setIsRowNumber(loRank.isRowNumber());

                poCounter.setIsDenseRank(loRank.isDenseRank());

                poCounter.setOperationID(String.valueOf(operationID));

                poRank = new PORank(

                        new OperatorKey(scope, nodeGen

                                .getNextNodeId(scope)), -1 , null,

                                rankPlans, loRank.getAscendingCol());

                poRank.addOriginalLocation(loRank.getAlias(), loRank.getLocation());

                poRank.setResultType(DataType.TUPLE);

                poRank.setOperationID(String.valueOf(operationID));

                currentPlan.add(poCounter);

                currentPlan.add(poRank);

                List<Operator> op = loRank.getPlan().getPredecessors(loRank);

                PhysicalOperator from;

                if(op != null) {

                    from = logToPhyMap.get(op.get(0));

                } else {

                    int errCode = 2051;

                    String msg = "Did not find a predecessor for Rank." ;

                    throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG);

                }

                currentPlan.connect(from, poCounter);

                currentPlan.connect(poCounter, poRank);

                logToPhyMap.put(loRank, poRank);

            }

        } catch (PlanException e) {

            int errCode = 2015;

            String msg = "Invalid physical operators in the physical plan" ;

            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);

        }

    }

    @Override

    public void visit(LOCross cross) throws FrontendException {

        String scope = DEFAULT_SCOPE;

        List<Operator> inputs = cross.getPlan().getPredecessors(cross);

                if (cross.isNested()) {

            POCross physOp = new POCross(new OperatorKey(scope,nodeGen.getNextNodeId(scope)), cross.getRequestedParallelism());

            physOp.addOriginalLocation(physOp.getAlias(), physOp.getOriginalLocations());

            currentPlan.add(physOp);

            physOp.setResultType(DataType.BAG);

            logToPhyMap.put(cross, physOp);

            for (Operator op : cross.getPlan().getPredecessors(cross)) {

                PhysicalOperator from = logToPhyMap.get(op);

                try {

                    currentPlan.connect(from, physOp);

                } catch (PlanException e) {

                    int errCode = 2015;

                    String msg = "Invalid physical operators in the physical plan" ;

                    throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);

                }

            }

        } else {

            POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(

                    scope, nodeGen.getNextNodeId(scope)), cross

                    .getRequestedParallelism());

            poGlobal.addOriginalLocation(cross.getAlias(), cross.getLocation());

            POPackage poPackage = new POPackage(new OperatorKey(scope, nodeGen

                    .getNextNodeId(scope)), cross.getRequestedParallelism());

            poGlobal.addOriginalLocation(cross.getAlias(), cross.getLocation());

            poGlobal.setCross(true);

            currentPlan.add(poGlobal);

            currentPlan.add(poPackage);

            int count = 0;

            try {

                currentPlan.connect(poGlobal, poPackage);

                List<Boolean> flattenLst = Arrays.asList(true, true);

                for (Operator op : inputs) {

                    PhysicalPlan fep1 = new PhysicalPlan();

                    ConstantExpression ce1 = new ConstantExpression(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),cross.getRequestedParallelism());

                    ce1.setValue(inputs.size());

                    ce1.setResultType(DataType.INTEGER);

                    fep1.add(ce1);

                    ConstantExpression ce2 = new ConstantExpression(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),cross.getRequestedParallelism());

                    ce2.setValue(count);

                    ce2.setResultType(DataType.INTEGER);

                    fep1.add(ce2);

                    /*Tuple ce1val = TupleFactory.getInstance().newTuple(2);

                    ce1val.set(0,inputs.size());

                    ce1val.set(1,count);

                    ce1.setValue(ce1val);

                    ce1.setResultType(DataType.TUPLE);*/

                    POUserFunc gfc = new POUserFunc(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),cross.getRequestedParallelism(),

                            Arrays.asList((PhysicalOperator)ce1,(PhysicalOperator)ce2), new FuncSpec(GFCross.class.getName()

                            + "('" + poGlobal.getOperatorKey().toString() + "')"));

                    gfc.addOriginalLocation(cross.getAlias(), cross.getLocation());

                    gfc.setResultType(DataType.BAG);

                    fep1.addAsLeaf(gfc);

                    gfc.setInputs(Arrays.asList((PhysicalOperator)ce1,(PhysicalOperator)ce2));

                    /*fep1.add(gfc);

                    fep1.connect(ce1, gfc);

                    fep1.connect(ce2, gfc);*/

                    PhysicalPlan fep2 = new PhysicalPlan();

                    POProject feproj = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelism());

                    feproj.addOriginalLocation(cross.getAlias(), cross.getLocation());

                    feproj.setResultType(DataType.TUPLE);

                    feproj.setStar(true);

                    feproj.setOverloaded(false);

                    fep2.add(feproj);

                    List<PhysicalPlan> fePlans = Arrays.asList(fep1, fep2);

                    POForEach fe = new POForEach(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelism(), fePlans, flattenLst );

                    fe.setMapSideOnly(true);

                    fe.addOriginalLocation(cross.getAlias(), cross.getLocation());

                    currentPlan.add(fe);

                    currentPlan.connect(logToPhyMap.get(op), fe);

                    POLocalRearrange physOp = new POLocalRearrange(new OperatorKey(

                            scope, nodeGen.getNextNodeId(scope)), cross

                            .getRequestedParallelism());

                    physOp.addOriginalLocation(cross.getAlias(), cross.getLocation());

                    List<PhysicalPlan> lrPlans = new ArrayList<PhysicalPlan>();

                    for(int i=0;i<inputs.size();i++){

                        PhysicalPlan lrp1 = new PhysicalPlan();

                        POProject lrproj1 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelism(), i);

                        lrproj1.addOriginalLocation(cross.getAlias(), cross.getLocation());

                        lrproj1.setOverloaded(false);

                        lrproj1.setResultType(DataType.INTEGER);

                        lrp1.add(lrproj1);

                        lrPlans.add(lrp1);

                    }

                    physOp.setCross(true);

                    physOp.setIndex(count++);

                    physOp.setKeyType(DataType.TUPLE);

                    physOp.setPlans(lrPlans);

                    physOp.setResultType(DataType.TUPLE);

                    currentPlan.add(physOp);

                    currentPlan.connect(fe, physOp);

                    currentPlan.connect(physOp, poGlobal);

                }

            } catch (PlanException e1) {

                int errCode = 2015;

                String msg = "Invalid physical operators in the physical plan" ;

                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e1);

            } catch (ExecException e) {

                int errCode = 2058;

                String msg = "Unable to set index on newly create POLocalRearrange.";

                throw new VisitorException(msg, errCode, PigException.BUG, e);

            }

            poPackage.getPkgr().setKeyType(DataType.TUPLE);

            poPackage.setResultType(DataType.TUPLE);

            poPackage.setNumInps(count);

            boolean inner[] = new boolean[count];

            for (int i=0;i<count;i++) {

                inner[i] = true;

            }

            poPackage.getPkgr().setInner(inner);

            List<PhysicalPlan> fePlans = new ArrayList<PhysicalPlan>();

            List<Boolean> flattenLst = new ArrayList<Boolean>();

            for(int i=1;i<=count;i++){

                PhysicalPlan fep1 = new PhysicalPlan();

                POProject feproj1 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelism(), i);

                feproj1.addOriginalLocation(cross.getAlias(), cross.getLocation());

                feproj1.setResultType(DataType.BAG);

                feproj1.setOverloaded(false);

                fep1.add(feproj1);

                fePlans.add(fep1);

                flattenLst.add(true);

            }

            POForEach fe = new POForEach(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelism(), fePlans, flattenLst );

            fe.addOriginalLocation(cross.getAlias(), cross.getLocation());

            currentPlan.add(fe);

            try{

                currentPlan.connect(poPackage, fe);

            }catch (PlanException e1) {

                int errCode = 2015;

                String msg = "Invalid physical operators in the physical plan" ;

                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e1);

            }

            logToPhyMap.put(cross, fe);

        }

    }

    @Override

    public void visit(LOStream stream) throws FrontendException {

        String scope = DEFAULT_SCOPE;

        POStream poStream = new POStream(new OperatorKey(scope, nodeGen

                .getNextNodeId(scope)), stream.getExecutableManager(),

                stream.getStreamingCommand(), this.pc.getProperties());

        poStream.addOriginalLocation(stream.getAlias(), stream.getLocation());

        currentPlan.add(poStream);

        logToPhyMap.put(stream, poStream);

        List<Operator> op = stream.getPlan().getPredecessors(stream);

        PhysicalOperator from;

        if(op != null) {

            from = logToPhyMap.get(op.get(0));

        } else {

            int errCode = 2051;

            String msg = "Did not find a predecessor for Stream." ;

            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG);

        }

        try {

            currentPlan.connect(from, poStream);

        } catch (PlanException e) {

            int errCode = 2015;

            String msg = "Invalid physical operators in the physical plan" ;

            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);

        }

    }

    @Override

    public void visit(LOInnerLoad load) throws FrontendException {

        String scope = DEFAULT_SCOPE;

        POProject exprOp = new POProject(new OperatorKey(scope, nodeGen

              .getNextNodeId(scope)));

        LogicalSchema s = load.getSchema();

        if (load.sourceIsBag()) {

            exprOp.setResultType(DataType.BAG);

            exprOp.setOverloaded(true);

        }

        else {

            if (s!=null)

                exprOp.setResultType(s.getField(0).type);

            else

                exprOp.setResultType(DataType.BYTEARRAY);

        }

        ProjectExpression proj = load.getProjection();

        if(proj.isProjectStar()){

            exprOp.setStar(proj.isProjectStar());

        }

        else if(proj.isRangeProject()){

            if(proj.getEndCol() != -1){

                //all other project-range should have been expanded by

                // project-star expander

                throw new AssertionError("project range that is not a " +

                "project-to-end seen in translation to physical plan!");

            }

            exprOp.setProjectToEnd(proj.getStartCol());

        }else {

            exprOp.setColumn(load.getColNum());

        }

        // set input to POProject to the predecessor of foreach

        logToPhyMap.put(load, exprOp);

        currentPlan.add(exprOp);

    }

    @Override

    public void visit(LOForEach foreach) throws FrontendException {

        String scope = DEFAULT_SCOPE;

        List<PhysicalPlan> innerPlans = new ArrayList<PhysicalPlan>();

        org.apache.pig.newplan.logical.relational.LogicalPlan inner = foreach.getInnerPlan();

        LOGenerate gen = (LOGenerate)inner.getSinks().get(0);

        List<LogicalExpressionPlan> exps = gen.getOutputPlans();

        List<Operator> preds = inner.getPredecessors(gen);

        currentPlans.push(currentPlan);

        // we need to translate each predecessor of LOGenerate into a physical plan.

        // The physical plan should contain the expression plan for this predecessor plus

        // the subtree starting with this predecessor

        for (int i=0; i<exps.size(); i++) {

            currentPlan = new PhysicalPlan();

            // translate the expression plan

            PlanWalker childWalker = new ReverseDependencyOrderWalkerWOSeenChk(exps.get(i));

            pushWalker(childWalker);

            childWalker.walk(new ExpToPhyTranslationVisitor(exps.get(i),

                    childWalker, gen, currentPlan, logToPhyMap));

            popWalker();

            List<Operator> leaves = exps.get(i).getSinks();

            for(Operator l: leaves) {

                PhysicalOperator op = logToPhyMap.get(l);

                if (l instanceof ProjectExpression ) {

                    int input = ((ProjectExpression)l).getInputNum();

                    // for each sink projection, get its input logical plan and translate it

                    Operator pred = preds.get(input);

                    childWalker = new SubtreeDependencyOrderWalker(inner, pred);

                    pushWalker(childWalker);

                    childWalker.walk(this);

                    popWalker();

                    // get the physical operator of the leaf of input logical plan

                    PhysicalOperator leaf = logToPhyMap.get(pred);

                    if (pred instanceof LOInnerLoad) {

                        // if predecessor is only an LOInnerLoad, remove the project that

                        // comes from LOInnerLoad and change the column of project that

                        // comes from expression plan

                        currentPlan.remove(leaf);

                        logToPhyMap.remove(pred);

                        POProject leafProj = (POProject)leaf;

                        try {

                            if(leafProj.isStar()){

                                ((POProject)op).setStar(true);

                            }

                            else if(leafProj.isProjectToEnd()){

                                ((POProject)op).setProjectToEnd(leafProj.getStartCol());

                            }else {

                                ((POProject)op).setColumn(leafProj.getColumn() );

                            }

                        } catch (ExecException e) {

                            throw new FrontendException(foreach, "Cannot get column from "+leaf, 2230, e);

                        }

                    }else{

                        currentPlan.connect(leaf, op);

                    }

                }

            }

            innerPlans.add(currentPlan);

        }

        currentPlan = currentPlans.pop();

        // PhysicalOperator poGen = new POGenerate(new OperatorKey("",

        // r.nextLong()), inputs, toBeFlattened);

        boolean[] flatten = gen.getFlattenFlags();

        List<Boolean> flattenList = new ArrayList<Boolean>();

        for(boolean fl: flatten) {

            flattenList.add(fl);

        }

        LogicalSchema logSchema = foreach.getSchema();

        Schema schema = null;

        if (logSchema != null) {

            try {

                schema = Schema.getPigSchema(new ResourceSchema(logSchema));

            } catch (FrontendException e) {

                throw new RuntimeException("LogicalSchema in foreach unable to be converted to Schema: " + logSchema, e);

            }

        }

        if (schema != null) {

            SchemaTupleFrontend.registerToGenerateIfPossible(schema, false, GenContext.FOREACH); //TODO may need to be appendable

        }

        POForEach poFE = new POForEach(new OperatorKey(scope, nodeGen

                .getNextNodeId(scope)), foreach.getRequestedParallelism(), innerPlans, flattenList, schema);

        poFE.addOriginalLocation(foreach.getAlias(), foreach.getLocation());

        poFE.setResultType(DataType.BAG);

        logToPhyMap.put(foreach, poFE);

        currentPlan.add(poFE);

        // generate cannot have multiple inputs

        List<Operator> op = foreach.getPlan().getPredecessors(foreach);

        // generate may not have any predecessors

        if (op == null)

            return;

        PhysicalOperator from = logToPhyMap.get(op.get(0));

        try {

           currentPlan.connect(from, poFE);

        } catch (Exception e) {

            int errCode = 2015;

            String msg = "Invalid physical operators in the physical plan" ;

            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);

        }

        translateSoftLinks(foreach);

    }

    /**

     * This function takes in a List of LogicalExpressionPlan and converts them to

     * a list of PhysicalPlans

     *

     * @param plans

     * @return

     * @throws FrontendException

     */

    private List<PhysicalPlan> translateExpressionPlans(LogicalRelationalOperator loj,

            List<LogicalExpressionPlan> plans ) throws FrontendException {

        List<PhysicalPlan> exprPlans = new ArrayList<PhysicalPlan>();

        if( plans == null || plans.size() == 0 ) {

            return exprPlans;

        }

        // Save the current plan onto stack

        currentPlans.push(currentPlan);

        for( LogicalExpressionPlan lp : plans ) {

            currentPlan = new PhysicalPlan();

            // We spawn a new Dependency Walker and use it

            // PlanWalker childWalker = currentWalker.spawnChildWalker(lp);

            PlanWalker childWalker = new ReverseDependencyOrderWalkerWOSeenChk(lp);

            // Save the old walker and use childWalker as current Walker

            pushWalker(childWalker);

            // We create a new ExpToPhyTranslationVisitor to walk the ExpressionPlan

            currentWalker.walk(

                    new ExpToPhyTranslationVisitor(

                            currentWalker.getPlan(),

                            childWalker, loj, currentPlan, logToPhyMap) );

            exprPlans.add(currentPlan);

            popWalker();

        }

        // Pop the current plan back out

        currentPlan = currentPlans.pop();

        return exprPlans;

    }

    @Override

    public void visit(LOStore loStore) throws FrontendException {

        String scope = DEFAULT_SCOPE;

        //        System.err.println("Entering Store");

        POStore store = new POStore(new OperatorKey(scope, nodeGen

                .getNextNodeId(scope)));

        store.addOriginalLocation(loStore.getAlias(), loStore.getLocation());

        store.setSFile(loStore.getOutputSpec());

        store.setInputSpec(loStore.getInputSpec());

        store.setSignature(loStore.getSignature());

        store.setSortInfo(loStore.getSortInfo());

        store.setIsTmpStore(loStore.isTmpStore());

        store.setStoreFunc(loStore.getStoreFunc());

        store.setSchema(Util.translateSchema( loStore.getSchema() ));

        if (loStore.getStoreFunc() instanceof StoreResources) {

            store.setCacheFiles(((StoreResources)loStore.getStoreFunc()).getCacheFiles());

            store.setShipFiles(((StoreResources)loStore.getStoreFunc()).getShipFiles());

        }

        currentPlan.add(store);

        List<Operator> op = loStore.getPlan().getPredecessors(loStore);

        PhysicalOperator from = null;

        if(op != null) {

            from = logToPhyMap.get(op.get(0));

//            SortInfo sortInfo = null;

//            // if store's predecessor is limit,

//            // check limit's predecessor

//            if(op.get(0) instanceof LOLimit) {

//                op = loStore.getPlan().getPredecessors(op.get(0));

//            }

//            PhysicalOperator sortPhyOp = logToPhyMap.get(op.get(0));

//            // if this predecessor is a sort, get

//            // the sort info.

//            if(op.get(0) instanceof LOSort) {

//                sortInfo = ((POSort)sortPhyOp).getSortInfo();

//            }

//            store.setSortInfo(sortInfo);

//        } else {

//            int errCode = 2051;

//            String msg = "Did not find a predecessor for Store." ;

//            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG);

        }

        try {

            currentPlan.connect(from, store);

        } catch (PlanException e) {

            int errCode = 2015;

            String msg = "Invalid physical operators in the physical plan" ;

            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);

        }

        logToPhyMap.put(loStore, store);

        //        System.err.println("Exiting Store");

    }

    @Override

    public void visit( LOCogroup cg ) throws FrontendException {

        switch (cg.getGroupType()) {

        case COLLECTED:

            translateCollectedCogroup(cg);

            break;

        case REGULAR:

            POPackage poPackage = compileToLR_GR_PackTrio(cg, cg.getCustomPartitioner(), cg.getInner(), cg.getExpressionPlans());

            poPackage.getPkgr().setPackageType(PackageType.GROUP);

            logToPhyMap.put(cg, poPackage);

            break;

        case MERGE:

            translateMergeCogroup(cg);

            break;

        default:

            throw new LogicalToPhysicalTranslatorException("Unknown CoGroup Modifier",PigException.BUG);

        }

        translateSoftLinks(cg);

    }

    private void translateCollectedCogroup(LOCogroup cg) throws FrontendException {

        // can have only one input

        LogicalRelationalOperator pred = (LogicalRelationalOperator) plan.getPredecessors(cg).get(0);

        List<LogicalExpressionPlan> exprPlans =  cg.getExpressionPlans().get(0);

        POCollectedGroup physOp = new POCollectedGroup(new OperatorKey(

                DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)));

        physOp.addOriginalLocation(cg.getAlias(), cg.getLocation());

        List<PhysicalPlan> pExprPlans = translateExpressionPlans(cg, exprPlans);