Examples of PODistinct

org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct
Find the distinct set of tuples in a bag. This is a blocking operator. All the input is put in the hashset implemented in DistinctDataBag which also provides the other DataBag interfaces.

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct

    }
    
    @Override
    public void visit(LODistinct loDistinct) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        PODistinct physOp = new PODistinct(new OperatorKey(scope,nodeGen.getNextNodeId(scope)), loDistinct.getRequestedParallelisam());
        physOp.setAlias(loDistinct.getAlias());
        currentPlan.add(physOp);
        physOp.setResultType(DataType.BAG);
        logToPhyMap.put(loDistinct, physOp);
        Operator op = loDistinct.getPlan().getPredecessors(loDistinct).get(0);


        PhysicalOperator from = logToPhyMap.get(op);
        try {

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct

        try {
            // Change PODistinct to use POSortedDistinct, which assume the input
            // data is sorted
            for (POToChange distinctToChange : distinctsToChange) {
                numDistinctChanged++;
                PODistinct oldDistinct = (PODistinct) distinctToChange.oper;
                String scope = oldDistinct.getOperatorKey().scope;
                POSortedDistinct newDistinct = new POSortedDistinct(
                        new OperatorKey(scope, NodeIdGenerator.getGenerator()
                                .getNextNodeId(scope)), oldDistinct
                                .getRequestedParallelism(), oldDistinct
                                .getInputs());
                newDistinct.setInputs(oldDistinct.getInputs());
                newDistinct.setResultType(oldDistinct.getResultType());
                distinctToChange.plan.replace(oldDistinct, newDistinct);
                distinctToChange.forEach.getLeaves();
            }
            // Removed POSort, if the successor require a databag, we need to
            // add a PORelationToExprProject

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct

    }
    
    @Override
    public void visit(LODistinct loDistinct) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        PODistinct physOp = new PODistinct(new OperatorKey(scope,nodeGen.getNextNodeId(scope)), loDistinct.getRequestedParallelisam());
        physOp.setAlias(loDistinct.getAlias());
        currentPlan.add(physOp);
        physOp.setResultType(DataType.BAG);
        logToPhyMap.put(loDistinct, physOp);
        Operator op = loDistinct.getPlan().getPredecessors(loDistinct).get(0);


        PhysicalOperator from = logToPhyMap.get(op);
        try {

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct

        try {
            // Change PODistinct to use POSortedDistinct, which assume the input
            // data is sorted
            for (POToChange distinctToChange : distinctsToChange) {
                numDistinctChanged++;
                PODistinct oldDistinct = (PODistinct) distinctToChange.oper;
                String scope = oldDistinct.getOperatorKey().scope;
                POSortedDistinct newDistinct = new POSortedDistinct(
                        new OperatorKey(scope, NodeIdGenerator.getGenerator()
                                .getNextNodeId(scope)), oldDistinct
                                .getRequestedParallelism(), oldDistinct
                                .getInputs());
                newDistinct.setInputs(oldDistinct.getInputs());
                newDistinct.setResultType(oldDistinct.getResultType());
                distinctToChange.plan.replace(oldDistinct, newDistinct);
                distinctToChange.forEach.getLeaves();
            }
            // Removed POSort, if the successor require a databag, we need to
            // add a PORelationToExprProject

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct

        try {
            // Change PODistinct to use POSortedDistinct, which assume the input
            // data is sorted
            for (POToChange distinctToChange : distinctsToChange) {
                numDistinctChanged++;
                PODistinct oldDistinct = (PODistinct) distinctToChange.oper;
                String scope = oldDistinct.getOperatorKey().scope;
                POSortedDistinct newDistinct = new POSortedDistinct(
                        new OperatorKey(scope, NodeIdGenerator.getGenerator()
                                .getNextNodeId(scope)), oldDistinct
                                .getRequestedParallelism(), oldDistinct
                                .getInputs());
                newDistinct.setInputs(oldDistinct.getInputs());
                newDistinct.setResultType(oldDistinct.getResultType());
                distinctToChange.plan.replace(oldDistinct, newDistinct);
                distinctToChange.forEach.getLeaves();
            }
            // Removed POSort, if the successor require a databag, we need to
            // add a PORelationToExprProject

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct

        try {
            // Change PODistinct to use POSortedDistinct, which assume the input
            // data is sorted
            for (POToChange distinctToChange : distinctsToChange) {
                secKeyOptimizerInfo.incrementNumDistinctChanged();
                PODistinct oldDistinct = (PODistinct) distinctToChange.oper;
                String scope = oldDistinct.getOperatorKey().scope;
                POSortedDistinct newDistinct = new POSortedDistinct(
                        new OperatorKey(scope, NodeIdGenerator.getGenerator()
                                .getNextNodeId(scope)), oldDistinct
                                .getRequestedParallelism(), oldDistinct
                                .getInputs());
                newDistinct.setInputs(oldDistinct.getInputs());
                newDistinct.setResultType(oldDistinct.getResultType());
                distinctToChange.plan.replace(oldDistinct, newDistinct);
                distinctToChange.forEach.getLeaves();
            }
            // Removed POSort, if the successor require a databag, we need to
            // add a PORelationToExprProject

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct

    }


    @Override
    public void visit(LODistinct loDistinct) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        PODistinct physOp = new PODistinct(new OperatorKey(scope,nodeGen.getNextNodeId(scope)), loDistinct.getRequestedParallelism());
        physOp.setCustomPartitioner(loDistinct.getCustomPartitioner());
        physOp.addOriginalLocation(loDistinct.getAlias(), loDistinct.getLocation());
        currentPlan.add(physOp);
        physOp.setResultType(DataType.BAG);
        logToPhyMap.put(loDistinct, physOp);
        Operator op = loDistinct.getPlan().getPredecessors(loDistinct).get(0);


        PhysicalOperator from = logToPhyMap.get(op);
        try {

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct

    }


    @Override
    public void visit(LODistinct loDistinct) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        PODistinct physOp = new PODistinct(new OperatorKey(scope,nodeGen.getNextNodeId(scope)), loDistinct.getRequestedParallelism());
        physOp.setCustomPartitioner(loDistinct.getCustomPartitioner());
        physOp.addOriginalLocation(loDistinct.getAlias(), loDistinct.getLocation());
        currentPlan.add(physOp);
        physOp.setResultType(DataType.BAG);
        logToPhyMap.put(loDistinct, physOp);
        Operator op = loDistinct.getPlan().getPredecessors(loDistinct).get(0);


        PhysicalOperator from = logToPhyMap.get(op);
        try {

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct

    public void testDistinct1() throws Exception {
        PhysicalPlan php = new PhysicalPlan();
        PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
        php.merge(ldFil1);


        PODistinct op = new PODistinct(new OperatorKey("", r.nextLong()),
                -1, null);


        php.addAsLeaf(op);


        PhysicalPlan grpChain1 = GenPhyOp.grpChain();
        php.merge(grpChain1);
        php.connect(op,grpChain1.getRoots().get(0));


        PODistinct op1 = new PODistinct(new OperatorKey("", r.nextLong()),
                -1, null);


        php.addAsLeaf(op1);
        POStore st = GenPhyOp.topStoreOp();
        php.addAsLeaf(st);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct

        PhysicalPlan grpChain1 = GenPhyOp.loadedGrpChain();
        php.merge(grpChain1);


        List<PhysicalPlan> inputs = new LinkedList<PhysicalPlan>();
        PhysicalPlan inplan = new PhysicalPlan();
        PODistinct op1 = new POSortedDistinct(new OperatorKey("", r.nextLong()),
                -1, null);
        inplan.addAsLeaf(op1);
        inputs.add(inplan);
        List<Boolean> toFlattens = new ArrayList<Boolean>();
        toFlattens.add(false);

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.