Examples of POProject


Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject

        List<PhysicalPlan> eps = new ArrayList<PhysicalPlan>();
       
        for (int i=0;i<join.getInputs().size();i++)
        {
            PhysicalPlan ep = new PhysicalPlan();
            POProject prj = new POProject(new OperatorKey(scope,nodeGen.getNextNodeId(scope)));
            prj.setResultType(DataType.BAG);
            prj.setColumn(i+1);
            prj.setOverloaded(false);
            prj.setStar(false);
            ep.add(prj);
            eps.add(ep);
            // the parser would have marked the side
            // where we need to keep empty bags on
            // non matched as outer (innerFlags[i] would be
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject

                new OperatorKey("", r.nextLong()), -1, null, new FuncSpec(funcName));
        POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, ldFil1.getLeaves(),
                null, new ArrayList<Boolean>(), comparator);
        sort.setRequestedParallelism(20);
        PhysicalPlan nesSortPlan = new PhysicalPlan();
        POProject topPrj = new POProject(new OperatorKey("", r.nextLong()));
        topPrj.setColumn(1);
        topPrj.setOverloaded(true);
        topPrj.setResultType(DataType.TUPLE);
        nesSortPlan.add(topPrj);
       
        POProject prjStar2 = new POProject(new OperatorKey("", r.nextLong()));
        prjStar2.setResultType(DataType.TUPLE);
        prjStar2.setStar(true);
        nesSortPlan.add(prjStar2);
       
        nesSortPlan.connect(topPrj, prjStar2);
        List<PhysicalPlan> nesSortPlanLst = new ArrayList<PhysicalPlan>();
        nesSortPlanLst.add(nesSortPlan);
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject

        PORead poread2 = new PORead(new OperatorKey("", r.nextLong()), bag2);

        List<PhysicalOperator> inputs1 = new LinkedList<PhysicalOperator>();
        inputs1.add(poread1);

        POProject prj1 = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
        prj1.setResultType(DataType.INTEGER);
        PhysicalPlan p1 = new PhysicalPlan();
        p1.add(prj1);
        List<PhysicalPlan> in1 = new LinkedList<PhysicalPlan>();
        in1.add(p1);
        POLocalRearrangeForIllustrate lr1 = new POLocalRearrangeForIllustrate(new OperatorKey("", r
                .nextLong()), -1, inputs1);
        lr1.setPlans(in1);
        lr1.setIndex(0);

        List<PhysicalOperator> inputs2 = new LinkedList<PhysicalOperator>();
        inputs2.add(poread2);

        POProject prj2 = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
        prj2.setResultType(DataType.INTEGER);
        PhysicalPlan p2 = new PhysicalPlan();
        p2.add(prj2);
        List<PhysicalPlan> in2 = new LinkedList<PhysicalPlan>();
        in2.add(p2);
        POLocalRearrangeForIllustrate lr2 = new POLocalRearrangeForIllustrate(new OperatorKey("", r
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject

        PORead poread1 = new PORead(new OperatorKey("", r.nextLong()), input);
        List<PhysicalOperator> inputs1 = new LinkedList<PhysicalOperator>();
        inputs1.add(poread1);

        POProject prj1 = new POProject(new OperatorKey("", r.nextLong()), -1, 1);
        prj1.setResultType(DataType.INTEGER);
        PhysicalPlan p1 = new PhysicalPlan();
        p1.add(prj1);
        List<PhysicalPlan> in1 = new LinkedList<PhysicalPlan>();
        in1.add(p1);
        POLocalRearrangeForIllustrate lr1 = new POLocalRearrangeForIllustrate(new OperatorKey("", r
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject

            if (op instanceof POForEach)
            {
                POForEach forEach = (POForEach)op;
                List<PhysicalPlan> planList = forEach.getInputPlans();
                List<Boolean> flatten = forEach.getToBeFlattened();
                POProject projOfLastInput = null;
                int i = 0;
                // check all nested foreach plans
                // 1. If it is simple projection
                // 2. If last input is all flattened
                for (PhysicalPlan p:planList)
                {
                    PhysicalOperator opProj = p.getRoots().get(0);
                    if (!(opProj instanceof POProject))
                    {
                        allSimple = false;
                        break;
                    }
                    POProject proj = (POProject)opProj;
                    // the project should just be for one column
                    // from the input
                    if(proj.getColumns().size() != 1) {
                        allSimple = false;
                        break;
                    }
                   
                    try {
                        // if input to project is the last input
                        if (proj.getColumn() == pack.getNumInps())
                        {
                            // if we had already seen another project
                            // which was also for the last input, then
                            // we might be trying to flatten twice on the
                            // last input in which case we can't optimize by
                            // just streaming the tuple to those projects
                            // IMPORTANT NOTE: THIS WILL NEED TO CHANGE WHEN WE
                            // OPTIMIZE BUILTINS LIKE SUM() AND COUNT() TO
                            // TAKE IN STREAMING INPUT
                            if(projOfLastInput != null) {
                                allSimple = false;
                                break;
                            }
                            projOfLastInput = proj;
                            // make sure the project is on a bag which needs to be
                            // flattened
                            if (!flatten.get(i) || proj.getResultType() != DataType.BAG)
                            {
                                lastInputFlattened = false;
                                break;
                            }
                        }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject

                columnChainInfo.insert(false, columns, DataType.TUPLE);

                PhysicalOperator node = plan.getRoots().get(0);
                while (node != null) {
                    if (node instanceof POProject) {
                        POProject project = (POProject) node;

                        columnChainInfo.insert(project.isStar(), project
                                .getColumns(), project.getResultType());

                        if (plan.getSuccessors(node) == null)
                            node = null;
                        else if (plan.getSuccessors(node).size() != 1) {
                            log.debug(node + " have more than 1 successor");
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject

                String scope = oldSort.getOperatorKey().scope;
                List<PhysicalOperator> preds = sortToRemove.plan
                        .getPredecessors(sortToRemove.oper);
                List<PhysicalOperator> succs = sortToRemove.plan
                .getSuccessors(sortToRemove.oper);
                POProject project = null;
                if ((preds == null
                        || preds.get(0).getResultType() != DataType.BAG
                        && oldSort.getResultType() == DataType.BAG) // sort to remove do change the result type
                        && (succs == null || !(succs.get(0) instanceof PORelationToExprProject))) // successor is not PORelationToExprProject
                {
                    project = new PORelationToExprProject(new OperatorKey(
                            scope, NodeIdGenerator.getGenerator()
                                    .getNextNodeId(scope)), oldSort
                            .getRequestedParallelism());
                    project.setInputs(oldSort.getInputs());
                    project.setResultType(DataType.BAG);
                    project.setStar(true);
                }
                if (project == null)
                    sortToRemove.plan.removeAndReconnect(sortToRemove.oper);
                else
                    sortToRemove.plan.replace(oldSort, project);
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject

                for (int i = 1; i < columnChainInfo.size(); i++) {
                    // The first item in columnChainInfo indicate the index of
                    // input, we have addressed
                    // already before we come here
                    ColumnInfo columnInfo = columnChainInfo.getColumnInfo(i);
                    POProject project = new POProject(
                            new OperatorKey(scope, NodeIdGenerator
                                    .getGenerator().getNextNodeId(scope)),
                            rearrange.getRequestedParallelism());
                    if (columnInfo.star)
                        project.setStar(true);
                    else
                        project
                                .setColumns((ArrayList<Integer>) columnInfo.columns);
                    project.setResultType(columnInfo.resultType);
                    secondaryPlan.addAsLeaf(project);
                }
                if (secondaryPlan.isEmpty()) { // If secondary key sort on the
                                               // input as a whole
                    POProject project = new POProject(
                            new OperatorKey(scope, NodeIdGenerator
                                    .getGenerator().getNextNodeId(scope)),
                            rearrange.getRequestedParallelism());
                    project.setStar(true);
                    secondaryPlan.addAsLeaf(project);
                }
                secondaryPlanList.add(secondaryPlan);
            }
            rearrange.setSecondaryPlans(secondaryPlanList);
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject

        PhysicalOperator currentNode = plan.getRoots().get(0);

        while (currentNode != null) {
            if (currentNode instanceof POProject) {
                POProject project = (POProject) currentNode;
                columnChainInfo.insertInReduce(project.isStar(), project
                        .getColumns(), project.getResultType());
            } else {
                return true;
            }
            List<PhysicalOperator> succs = plan.getSuccessors(currentNode);
            if (succs == null)
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject

    }
   
    private void simpleConnectMapToReduce(MapReduceOper mro) throws PlanException
    {
        PhysicalPlan ep = new PhysicalPlan();
        POProject prjStar = new POProject(new OperatorKey(scope,nig.getNextNodeId(scope)));
        prjStar.setResultType(DataType.TUPLE);
        prjStar.setStar(true);
        ep.add(prjStar);
       
        List<PhysicalPlan> eps = new ArrayList<PhysicalPlan>();
        eps.add(ep);
       
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.