Examples of POLocalRearrange

org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange
The local rearrange operator is a part of the co-group implementation. It has an embedded physical plan that generates tuples of the form (grpKey,(indxed inp Tuple)).

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange

        assertEquals( 1, phyPlan.getRoots().size() );
        assertEquals( POLoad.class, phyPlan.getRoots().get(0).getClass() );
        POLoad load = (POLoad)phyPlan.getRoots().get(0);
        
        assertEquals( POLocalRearrange.class, phyPlan.getSuccessors(load).get(0).getClass() );
        POLocalRearrange localR = (POLocalRearrange)phyPlan.getSuccessors(load).get(0);
        assertEquals( 1, localR.getInputs().size() );
        assertEquals( 2, localR.getPlans().size() );
        PhysicalPlan cogroupPlan = localR.getPlans().get(0);
        assertEquals( 1, cogroupPlan.getLeaves().size() );        
        assertEquals( POProject.class, cogroupPlan.getLeaves().get(0).getClass() );
        POProject prj = (POProject)cogroupPlan.getLeaves().get(0);
        assertEquals( 0, prj.getColumn() );
        assertEquals( DataType.CHARARRAY, prj.getResultType() );
        
        PhysicalPlan cogroupPlan2 = localR.getPlans().get(1);
        POProject prj2 = (POProject)cogroupPlan2.getLeaves().get(0);
        assertEquals( 1, prj2.getColumn() );
        assertEquals( DataType.INTEGER, prj2.getResultType() );
        
        assertEquals( POGlobalRearrange.class, phyPlan.getSuccessors(localR).get(0).getClass() );

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange

        assertEquals( 2, phyPlan.getRoots().size() );
        assertEquals( POLoad.class, phyPlan.getRoots().get(0).getClass() );
        POLoad load = (POLoad)phyPlan.getRoots().get(0);
        
        assertEquals( POLocalRearrange.class, phyPlan.getSuccessors(load).get(0).getClass() );
        POLocalRearrange localR = (POLocalRearrange)phyPlan.getSuccessors(load).get(0);
        assertEquals( 1, localR.getPlans().size() );
        PhysicalPlan cogroupPlan = localR.getPlans().get(0);
        assertEquals( 1, cogroupPlan.getLeaves().size() );        
        assertEquals( POProject.class, cogroupPlan.getLeaves().get(0).getClass() );
        POProject prj = (POProject)cogroupPlan.getLeaves().get(0);
        assertEquals( 0, prj.getColumn() );
        assertEquals( DataType.CHARARRAY, prj.getResultType() );
        
        assertEquals( POGlobalRearrange.class, phyPlan.getSuccessors(localR).get(0).getClass() );
        POGlobalRearrange globalR = (POGlobalRearrange)phyPlan.getSuccessors(localR).get(0);
        assertEquals( DataType.TUPLE, globalR.getResultType() );
        
        assertEquals( POLoad.class, phyPlan.getRoots().get(1).getClass() );
        POLoad load2 = (POLoad)phyPlan.getRoots().get(0);
        
        assertEquals( POLocalRearrange.class, phyPlan.getSuccessors(load2).get(0).getClass() );
        POLocalRearrange localR2 = (POLocalRearrange)phyPlan.getSuccessors(load2).get(0);
        assertEquals( 1, localR2.getPlans().size() );
        PhysicalPlan cogroupPlan2 = localR2.getPlans().get(0);
        POProject prj2 = (POProject)cogroupPlan2.getLeaves().get(0);
        assertEquals( 0, prj2.getColumn() );
        assertEquals( DataType.CHARARRAY, prj2.getResultType() );
        
        assertEquals( POPackage.class, phyPlan.getSuccessors(globalR).get(0).getClass() );

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange

        assertEquals( 2, phyPlan.getRoots().size() );
        assertEquals( POLoad.class, phyPlan.getRoots().get(0).getClass() );
        POLoad load = (POLoad)phyPlan.getRoots().get(0);
        
        assertEquals( POLocalRearrange.class, phyPlan.getSuccessors(load).get(0).getClass() );
        POLocalRearrange localR = (POLocalRearrange)phyPlan.getSuccessors(load).get(0);
        assertEquals( 2, localR.getPlans().size() );
        PhysicalPlan cogroupPlan = localR.getPlans().get(0);
        assertEquals( 1, cogroupPlan.getLeaves().size() );        
        assertEquals( POProject.class, cogroupPlan.getLeaves().get(0).getClass() );
        POProject prj = (POProject)cogroupPlan.getLeaves().get(0);
        assertEquals( 0, prj.getColumn() );
        assertEquals( DataType.CHARARRAY, prj.getResultType() );
        
        PhysicalPlan cogroupPlan2 = localR.getPlans().get(1);
        assertEquals( 1, cogroupPlan2.getLeaves().size() );        
        assertEquals( POProject.class, cogroupPlan2.getLeaves().get(0).getClass() );
        POProject prj2 = (POProject)cogroupPlan2.getLeaves().get(0);
        assertEquals( 1, prj2.getColumn() );
        assertEquals( DataType.INTEGER, prj2.getResultType() );
        
        assertEquals( POGlobalRearrange.class, phyPlan.getSuccessors(localR).get(0).getClass() );
        POGlobalRearrange globalR = (POGlobalRearrange)phyPlan.getSuccessors(localR).get(0);
        assertEquals( DataType.TUPLE, globalR.getResultType() );
        
        assertEquals( POLoad.class, phyPlan.getRoots().get(1).getClass() );
        POLoad load2 = (POLoad)phyPlan.getRoots().get(0);
        
        assertEquals( POLocalRearrange.class, phyPlan.getSuccessors(load2).get(0).getClass() );
        
        POLocalRearrange localR3 = (POLocalRearrange)phyPlan.getSuccessors(load2).get(0);
        assertEquals( 2, localR3.getPlans().size() );
        PhysicalPlan cogroupPlan3 = localR3.getPlans().get(0);
        POProject prj3 = (POProject)cogroupPlan3.getLeaves().get(0);
        assertEquals( 0, prj3.getColumn() );
        assertEquals( DataType.CHARARRAY, prj3.getResultType() );
        
        PhysicalPlan cogroupPlan4 = localR3.getPlans().get(1);
        POProject prj4 = (POProject)cogroupPlan4.getLeaves().get(0);
        assertEquals( 1, prj4.getColumn() );
        assertEquals( DataType.INTEGER, prj4.getResultType() );
        
        assertEquals( POPackage.class, phyPlan.getSuccessors(globalR).get(0).getClass() );

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange

        
        List<PhysicalPlan> pls = splitOp.getPlans();
        for (PhysicalPlan pl : pls) {
            PhysicalOperator leaf = pl.getLeaves().get(0);
            if (leaf instanceof POLocalRearrange) {
                POLocalRearrange lr = (POLocalRearrange)leaf;
                try {
                    lr.setMultiQueryIndex(index++); 
                } catch (ExecException e) {                    
                    int errCode = 2136;
                    String msg = "Internal Error. Unable to set multi-query index for optimization.";
                    throw new OptimizerException(msg, errCode, PigException.BUG, e);                   
                }
                
                // change the map key type to tuple when 
                // multiple splittees have different map key types
                if (!sameKeyType) {
                    lr.setKeyType(DataType.TUPLE);
                }
            } else if (leaf instanceof POSplit) {
                POSplit spl = (POSplit)leaf;
                index = setIndexOnLRInSplit(index, spl, sameKeyType);
            }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange

                
        int curIndex = index;
        
        PhysicalOperator leaf = pl.getLeaves().get(0);
        if (leaf instanceof POLocalRearrange) {
            POLocalRearrange lr = (POLocalRearrange)leaf;
            try {
                lr.setMultiQueryIndex(curIndex++);  
            } catch (ExecException e) {                                      
                int errCode = 2136;
                String msg = "Internal Error. Unable to set multi-query index for optimization.";
                throw new OptimizerException(msg, errCode, PigException.BUG, e);
            }
            
            // change the map key type to tuple when 
            // multiple splittees have different map key types
            if (!sameKeyType) {
                lr.setKeyType(DataType.TUPLE);
            }
        } else if (leaf instanceof POSplit) {
            // if the map plan that we are trying to merge
            // has a split, we need to update the indices of
            // the POLocalRearrange operators in the inner plans

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange

        PhysicalOperator leaf = from.getLeaves().get(0);
        if (leaf instanceof PODemux) {
            List<PhysicalPlan> pls = ((PODemux)leaf).getPlans();
            for (PhysicalPlan pl : pls) {
                demux.addPlan(pl);
                POLocalRearrange lr = (POLocalRearrange)pl.getLeaves().get(0);
                try {
                    lr.setMultiQueryIndex(initial + plCount++);            
                } catch (ExecException e) {                                        
                    int errCode = 2136;
                    String msg = "Internal Error. Unable to set multi-query index for optimization.";
                    throw new OptimizerException(msg, errCode, PigException.BUG, e);
                }
                
                // change the map key type to tuple when 
                // multiple splittees have different map key types
                if (!isSameKeyType) {
                    lr.setKeyType(DataType.TUPLE);
                }
            }
        } else {
            demux.addPlan(from);
            POLocalRearrange lr = (POLocalRearrange)from.getLeaves().get(0);
            try {
                lr.setMultiQueryIndex(initial + plCount++);            
            } catch (ExecException e) {                                        
                int errCode = 2136;
                String msg = "Internal Error. Unable to set multi-query index for optimization.";
                throw new OptimizerException(msg, errCode, PigException.BUG, e);
            }
                
            // change the map key type to tuple when 
            // multiple splittees have different map key types
            if (!isSameKeyType) {
                lr.setKeyType(DataType.TUPLE);
            }
        }
        
        if (plCount != total) {
            int errCode = 2146;

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange

        assertEquals(physicalPlan.getRoots().size(), 2);
        
        // Check Load and LocalRearrange and GlobalRearrange
        PhysicalOperator LoR = (PhysicalOperator)physicalPlan.getSuccessors(physicalPlan.getRoots().get(0)).get(0);
        assertEquals( POLocalRearrange.class, LoR.getClass() );
        POLocalRearrange Lor = (POLocalRearrange) LoR;
        PhysicalOperator prj3 = Lor.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj3.getClass() );
        assertEquals(0, ((POProject)prj3).getColumn() );
        PhysicalOperator inp1 = Lor.getInputs().get(0);
        assertEquals( POLoad.class, inp1.getClass() );
        assertTrue(  ((POLoad)inp1).getLFile().getFileName().contains("d1.txt") );
                
        PhysicalOperator LoR1 = (PhysicalOperator)physicalPlan.getSuccessors(physicalPlan.getRoots().get(1)).get(0);
        assertEquals( POLocalRearrange.class, LoR1.getClass() );
        POLocalRearrange Lor1 = (POLocalRearrange) LoR1;
        PhysicalOperator prj4 = Lor1.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj4.getClass() );
        assertEquals(1, ((POProject)prj4).getColumn() );
        PhysicalOperator inp2 = Lor1.getInputs().get(0);
        assertEquals( POLoad.class, inp2.getClass() );
        assertTrue(  ((POLoad)inp2).getLFile().getFileName().contains("d2.txt") );
        
        PhysicalOperator GoR = (PhysicalOperator)physicalPlan.getSuccessors(LoR).get(0);
        assertEquals( POGlobalRearrange.class, GoR.getClass() );

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange

        assertEquals(phyPlan.getLeaves().size(), 1 );


        // Check Load and LocalRearrange and GlobalRearrange
        PhysicalOperator LoR = (PhysicalOperator)phyPlan.getSuccessors(phyPlan.getRoots().get(0)).get(0);
        assertEquals( POLocalRearrange.class, LoR.getClass() );
        POLocalRearrange Lor = (POLocalRearrange) LoR;
        PhysicalOperator prj1 = Lor.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj1.getClass() );
        assertEquals(0, ((POProject)prj1).getColumn() );
        PhysicalOperator inp1 = Lor.getInputs().get(0);
        assertEquals( POLoad.class, inp1.getClass() );
        assertTrue(  ((POLoad)inp1).getLFile().getFileName().contains("d3.txt") );


        PhysicalOperator LoR1 = (PhysicalOperator)phyPlan.getSuccessors(phyPlan.getRoots().get(1)).get(0);
        assertEquals( POLocalRearrange.class, LoR1.getClass() );
        POLocalRearrange Lor1 = (POLocalRearrange) LoR1;
        PhysicalOperator prj2 = Lor1.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj2.getClass() );
        assertEquals(1, ((POProject)prj2).getColumn() );
        PhysicalOperator inp2 = Lor1.getInputs().get(0);
        assertEquals( POLoad.class, inp2.getClass() );
        assertTrue(  ((POLoad)inp2).getLFile().getFileName().contains("d2.txt") );
        
        PhysicalOperator GoR = (PhysicalOperator)phyPlan.getSuccessors(LoR).get(0);
        assertEquals( POGlobalRearrange.class, GoR.getClass() );
        
        PhysicalOperator Pack = (PhysicalOperator)phyPlan.getSuccessors(GoR).get(0);
        assertEquals( POPackage.class, Pack.getClass() );
        
        PhysicalOperator LoR2 = (PhysicalOperator)phyPlan.getSuccessors(phyPlan.getRoots().get(2)).get(0);
        assertEquals( POLocalRearrange.class, LoR2.getClass() );
        POLocalRearrange Lor2 = (POLocalRearrange) LoR2;
        PhysicalOperator prj3 = Lor2.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj3.getClass() );
        assertEquals(0, ((POProject)prj3).getColumn() );
        PhysicalOperator inp3 = Lor2.getInputs().get(0);
        assertEquals( POLoad.class, inp3.getClass() );
        assertTrue(  ((POLoad)inp3).getLFile().getFileName().contains("d1.txt") );
        
        PhysicalOperator GoR2 = (PhysicalOperator)phyPlan.getSuccessors(LoR2).get(0);
        assertEquals( POGlobalRearrange.class, GoR2.getClass() );
        
        PhysicalOperator Pack2 = (PhysicalOperator)phyPlan.getSuccessors(GoR2).get(0);
        assertEquals( POPackage.class, Pack2.getClass() );
        
        // Check for ForEach
        PhysicalOperator ForE = (PhysicalOperator)phyPlan.getSuccessors(Pack).get(0);
        assertEquals( POForEach.class, ForE.getClass() );
        PhysicalOperator prj4 = ((POForEach)ForE).getInputPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj4.getClass() );
        assertEquals( 1, ((POProject)prj4).getColumn() ); 
        PhysicalOperator prj5 = ((POForEach)ForE).getInputPlans().get(1).getLeaves().get(0);
        assertEquals( POProject.class, prj5.getClass() );
        assertEquals( 2, ((POProject)prj5).getColumn() );
        
        PhysicalOperator ForE2 = (PhysicalOperator)phyPlan.getSuccessors(Pack2).get(0);
        assertEquals( POForEach.class, ForE2.getClass() );
        PhysicalOperator prj6 = ((POForEach)ForE2).getInputPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj6.getClass() );
        assertEquals( 1, ((POProject)prj6).getColumn() ); 
        PhysicalOperator prj7 = ((POForEach)ForE2).getInputPlans().get(1).getLeaves().get(0);
        assertEquals( POProject.class, prj7.getClass() );
        assertEquals( 2, ((POProject)prj7).getColumn() );
        
        // Check Filter Operator
        PhysicalOperator fil = (PhysicalOperator)phyPlan.getSuccessors(ForE).get(0);
        assertEquals( POFilter.class, fil.getClass() );
        
        PhysicalPlan filPlan = ((POFilter)fil).getPlan();
        List<PhysicalOperator> filRoots = filPlan.getRoots();
        
        assertEquals( ConstantExpression.class, filRoots.get(0).getClass() );
        ConstantExpression ce1 = (ConstantExpression) filRoots.get(0);
        assertEquals( null, ce1.getValue() ); 
        assertEquals( ConstantExpression.class, filRoots.get(2).getClass() );
        ConstantExpression ce2 = (ConstantExpression) filRoots.get(2);
        assertEquals( null, ce2.getValue() );
        assertEquals( POProject.class, filRoots.get(1).getClass() );
        POProject prj8 = (POProject) filRoots.get(1);
        assertEquals( 5, prj8.getColumn() );
        assertEquals( POProject.class, filRoots.get(3).getClass() );
        POProject prj9 = (POProject) filRoots.get(3);
        assertEquals( 2, prj9.getColumn() );
        
        
        PhysicalOperator fil2 = (PhysicalOperator)phyPlan.getSuccessors(ForE2).get(0);
        assertEquals( POFilter.class, fil2.getClass() );
        
        PhysicalOperator LoR3 = (PhysicalOperator)phyPlan.getSuccessors(fil2).get(0);
        assertEquals( POLocalRearrange.class, LoR3.getClass() );
        POLocalRearrange Lor3 = (POLocalRearrange) LoR3;
        PhysicalOperator prj12 = Lor3.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj12.getClass() );
        assertEquals(3, ((POProject)prj12).getColumn() );
        
        PhysicalPlan filPlan2 = ((POFilter)fil2).getPlan();
        List<PhysicalOperator> filRoots2 = filPlan2.getRoots();

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange

        assertEquals( 1, phyPlan.getRoots().size() );
        assertEquals( POLoad.class, phyPlan.getRoots().get(0).getClass() );
        POLoad load = (POLoad)phyPlan.getRoots().get(0);
        
        assertEquals( POLocalRearrange.class, phyPlan.getSuccessors(load).get(0).getClass() );
        POLocalRearrange localR = (POLocalRearrange)phyPlan.getSuccessors(load).get(0);
        assertEquals( 1, localR.getInputs().size() );
        assertEquals( 1, localR.getPlans().size() );
        PhysicalPlan cogroupPlan = localR.getPlans().get(0);
        assertEquals( 1, cogroupPlan.getLeaves().size() );        
        assertEquals( POProject.class, cogroupPlan.getLeaves().get(0).getClass() );
        POProject prj = (POProject)cogroupPlan.getLeaves().get(0);
        assertEquals( 0, prj.getColumn() );
        assertEquals( DataType.CHARARRAY, prj.getResultType() );

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange

        assertEquals( 1, phyPlan.getRoots().size() );
        assertEquals( POLoad.class, phyPlan.getRoots().get(0).getClass() );
        POLoad load = (POLoad)phyPlan.getRoots().get(0);
        
        assertEquals( POLocalRearrange.class, phyPlan.getSuccessors(load).get(0).getClass() );
        POLocalRearrange localR = (POLocalRearrange)phyPlan.getSuccessors(load).get(0);
        assertEquals( 1, localR.getInputs().size() );
        assertEquals( 2, localR.getPlans().size() );
        PhysicalPlan cogroupPlan = localR.getPlans().get(0);
        assertEquals( 1, cogroupPlan.getLeaves().size() );        
        assertEquals( POProject.class, cogroupPlan.getLeaves().get(0).getClass() );
        POProject prj = (POProject)cogroupPlan.getLeaves().get(0);
        assertEquals( 0, prj.getColumn() );
        assertEquals( DataType.CHARARRAY, prj.getResultType() );
        
        PhysicalPlan cogroupPlan2 = localR.getPlans().get(1);
        POProject prj2 = (POProject)cogroupPlan2.getLeaves().get(0);
        assertEquals( 1, prj2.getColumn() );
        assertEquals( DataType.INTEGER, prj2.getResultType() );
        
        assertEquals( POGlobalRearrange.class, phyPlan.getSuccessors(localR).get(0).getClass() );

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.