Examples of POSort


Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

            // Removed POSort, if the successor require a databag, we need to
            // add a PORelationToExprProject
            // to convert tuples into databag
            for (POToChange sortToRemove : sortsToRemove) {
                secKeyOptimizerInfo.incrementNumSortRemoved();
                POSort oldSort = (POSort) sortToRemove.oper;
                String scope = oldSort.getOperatorKey().scope;
                List<PhysicalOperator> preds = sortToRemove.plan
                        .getPredecessors(sortToRemove.oper);
                List<PhysicalOperator> succs = sortToRemove.plan
                .getSuccessors(sortToRemove.oper);
                POProject project = null;
                if ((preds == null
                        || preds.get(0).getResultType() != DataType.BAG
                        && oldSort.getResultType() == DataType.BAG) // sort to remove do change the result type
                        && (succs == null || !(succs.get(0) instanceof PORelationToExprProject))) // successor is not PORelationToExprProject
                {
                    project = new PORelationToExprProject(new OperatorKey(
                            scope, NodeIdGenerator.getGenerator()
                                    .getNextNodeId(scope)), oldSort
                            .getRequestedParallelism());
                    project.setInputs(oldSort.getInputs());
                    project.setResultType(DataType.BAG);
                    project.setStar(true);
                }
                if (project == null)
                    sortToRemove.plan.removeAndReconnect(sortToRemove.oper);
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

            popWalker();
        }
        currentPlan = currentPlans.pop();

        // get the physical operator for sort
        POSort poSort;
        if (sort.getUserFunc() == null) {
            poSort = new POSort(new OperatorKey(scope, nodeGen
                    .getNextNodeId(scope)), sort.getRequestedParallelism(), null,
                    sortPlans, sort.getAscendingCols(), null);
        } else {
            POUserComparisonFunc comparator = new POUserComparisonFunc(new OperatorKey(
                    scope, nodeGen.getNextNodeId(scope)), sort
                    .getRequestedParallelism(), null, sort.getUserFunc());
            poSort = new POSort(new OperatorKey(scope, nodeGen
                    .getNextNodeId(scope)), sort.getRequestedParallelism(), null,
                    sortPlans, sort.getAscendingCols(), comparator);
        }
        poSort.addOriginalLocation(sort.getAlias(), sort.getLocation());
        poSort.setLimit(sort.getLimit());
        // sort.setRequestedParallelism(s.getType());
        logToPhyMap.put(sort, poSort);
        currentPlan.add(poSort);
        List<Operator> op = sort.getPlan().getPredecessors(sort);
        PhysicalOperator from;

        if(op != null) {
            from = logToPhyMap.get(op.get(0));
        } else {
            int errCode = 2051;
            String msg = "Did not find a predecessor for Sort." ;
            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG);
        }

        try {
            currentPlan.connect(from, poSort);
        } catch (PlanException e) {
            int errCode = 2015;
            String msg = "Invalid physical operators in the physical plan" ;
            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
        }

        poSort.setResultType(DataType.BAG);
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

                    newPhysicalPlan.add(fep3);
                    newOrderPlan.add(loRank.getAscendingCol().get(i));
                }

                POSort poSort;
                poSort = new POSort(new OperatorKey(scope, nodeGen
                        .getNextNodeId(scope)), -1, null,
                        newPhysicalPlan, newOrderPlan, null);
                poSort.addOriginalLocation(loRank.getAlias(), loRank.getLocation());


                poCounter = new POCounter(
                        new OperatorKey(scope, nodeGen
                                .getNextNodeId(scope)), -1 , null,
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

            MapReduceOper prevJob,
            FileSpec lFile,
            FileSpec quantFile,
            int rp) throws PlanException, VisitorException {

        POSort sort = new POSort(inpSort.getOperatorKey(), inpSort
                .getRequestedParallelism(), null, inpSort.getSortPlans(),
                inpSort.getMAscCols(), inpSort.getMSortFunc());
        sort.addOriginalLocation(inpSort.getAlias(), inpSort.getOriginalLocations());

        // Turn the asc/desc array into an array of strings so that we can pass it
        // to the FindQuantiles function.
        List<Boolean> ascCols = inpSort.getMAscCols();
        String[] ascs = new String[ascCols.size()];
        for (int i = 0; i < ascCols.size(); i++) ascs[i] = ascCols.get(i).toString();
        // check if user defined comparator is used in the sort, if so
        // prepend the name of the comparator as the first fields in the
        // constructor args array to the FindQuantiles udf
        String[] ctorArgs = ascs;
        if(sort.isUDFComparatorUsed) {
            String userComparatorFuncSpec = sort.getMSortFunc().getFuncSpec().toString();
            ctorArgs = new String[ascs.length + 1];
            ctorArgs[0] = USER_COMPARATOR_MARKER + userComparatorFuncSpec;
            for(int j = 0; j < ascs.length; j++) {
                ctorArgs[j+1] = ascs[j];
            }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

        List<Boolean> ascCol = new ArrayList<Boolean>();
        for(int i=0; i<groups.size(); i++) {
            ascCol.add(false);
        }

        POSort sort = new POSort(op.getOperatorKey(), op.getRequestedParallelism(), null, groups, ascCol, null);

        // set up transform plan to get keys and memory size of input tuples
        // it first adds all the plans to get key columns,
        List<PhysicalPlan> transformPlans = new ArrayList<PhysicalPlan>();
        transformPlans.addAll(groups);
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

            popWalker();
        }
        currentPlan = currentPlans.pop();

        // get the physical operator for sort
        POSort poSort;
        if (sort.getUserFunc() == null) {
            poSort = new POSort(new OperatorKey(scope, nodeGen
                    .getNextNodeId(scope)), sort.getRequestedParallelism(), null,
                    sortPlans, sort.getAscendingCols(), null);
        } else {
            POUserComparisonFunc comparator = new POUserComparisonFunc(new OperatorKey(
                    scope, nodeGen.getNextNodeId(scope)), sort
                    .getRequestedParallelism(), null, sort.getUserFunc());
            poSort = new POSort(new OperatorKey(scope, nodeGen
                    .getNextNodeId(scope)), sort.getRequestedParallelism(), null,
                    sortPlans, sort.getAscendingCols(), comparator);
        }
        poSort.addOriginalLocation(sort.getAlias(), sort.getLocation());
        poSort.setLimit(sort.getLimit());
        // sort.setRequestedParallelism(s.getType());
        logToPhyMap.put(sort, poSort);
        currentPlan.add(poSort);
        List<Operator> op = sort.getPlan().getPredecessors(sort);
        PhysicalOperator from;

        if(op != null) {
            from = logToPhyMap.get(op.get(0));
        } else {
            int errCode = 2051;
            String msg = "Did not find a predecessor for Sort." ;
            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG);
        }

        try {
            currentPlan.connect(from, poSort);
        } catch (PlanException e) {
            int errCode = 2015;
            String msg = "Invalid physical operators in the physical plan" ;
            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
        }

        poSort.setResultType(DataType.BAG);
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

                    newPhysicalPlan.add(fep3);
                    newOrderPlan.add(loRank.getAscendingCol().get(i));
                }

                POSort poSort;
                poSort = new POSort(new OperatorKey(scope, nodeGen
                        .getNextNodeId(scope)), -1, null,
                        newPhysicalPlan, newOrderPlan, null);
                poSort.addOriginalLocation(loRank.getAlias(), loRank.getLocation());


                poCounter = new POCounter(
                        new OperatorKey(scope, nodeGen
                                .getNextNodeId(scope)), -1 , null,
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

        // set up order by *
        String funcName = WeirdComparator.class.getName();
        POUserComparisonFunc comparator = new POUserComparisonFunc(
                new OperatorKey("", r.nextLong()), -1, null, new FuncSpec(funcName));
        POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, ldFil1.getLeaves(),
                null, new ArrayList<Boolean>(), comparator);
        sort.setRequestedParallelism(20);
        PhysicalPlan nesSortPlan = new PhysicalPlan();
        POProject topPrj = new POProject(new OperatorKey("", r.nextLong()));
        topPrj.setColumn(1);
        topPrj.setOverloaded(true);
        topPrj.setResultType(DataType.TUPLE);
        nesSortPlan.add(topPrj);

        POProject prjStar2 = new POProject(new OperatorKey("", r.nextLong()));
        prjStar2.setResultType(DataType.TUPLE);
        prjStar2.setStar(true);
        nesSortPlan.add(prjStar2);

        nesSortPlan.connect(topPrj, prjStar2);
        List<PhysicalPlan> nesSortPlanLst = new ArrayList<PhysicalPlan>();
        nesSortPlanLst.add(nesSortPlan);

        sort.setSortPlans(nesSortPlanLst);

        php.add(sort);
        php.connect(ldFil1.getLeaves().get(0), sort);
        // have a foreach which takes the sort output
        // and send it two two udfs
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

        List<Boolean> mAscCols = new LinkedList<Boolean>();
        mAscCols.add(true);
        PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
        List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
        inputs.add(read);
        POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
                sortPlans, mAscCols, null);

        //verify
        Tuple t = null;
        Result res1 = sort.getNextTuple();
        Result res2 = sort.getNextTuple();

        while (res2.returnStatus != POStatus.STATUS_EOP) {
            Object i1 = ((Tuple) res1.result).get(0);
            Object i2 = ((Tuple) res2.result).get(0);

            //System.out.println("i1: " + i1.toString() + " i2: " + i2.toString());
            int i = DataType.compare(i1, i2);
            System.out.println("RESULT2=i : " + res2.result + " i = " + i);
            assertEquals(true, (i <= 0));
            res1 = res2;
            res2 = sort.getNextTuple();
        }
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort

        List<Boolean> mAscCols = new LinkedList<Boolean>();
        mAscCols.add(false);
        PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
        List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
        inputs.add(read);
        POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
                sortPlans, mAscCols, null);
        Tuple t = null;
        Result res1 = sort.getNextTuple();
        // System.out.println(res1.result);
        Result res2 = sort.getNextTuple();
        while (res2.returnStatus != POStatus.STATUS_EOP) {
            Object i1 = ((Tuple) res1.result).get(0);
            Object i2 = ((Tuple) res2.result).get(0);
            int i = DataType.compare(i1, i2);
            // System.out.println(res2.result + " i = " + i);
            assertEquals(true, (i >= 0));
            res1 = res2;
            res2 = sort.getNextTuple();
        }
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.