Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject

Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject

org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject
Implements the overloaded form of the project operator. Projects the specified column from the input tuple. However, if asked for tuples when the input is a bag, the overloaded form is invoked and the project streams the tuples through instead of the bag.

        assertEquals( 1, localR.getInputs().size() );
        assertEquals( 2, localR.getPlans().size() );
        PhysicalPlan cogroupPlan = localR.getPlans().get(0);
        assertEquals( 1, cogroupPlan.getLeaves().size() );        
        assertEquals( POProject.class, cogroupPlan.getLeaves().get(0).getClass() );
        POProject prj = (POProject)cogroupPlan.getLeaves().get(0);
        assertEquals( 0, prj.getColumn() );
        assertEquals( DataType.CHARARRAY, prj.getResultType() );
        
        PhysicalPlan cogroupPlan2 = localR.getPlans().get(1);
        POProject prj2 = (POProject)cogroupPlan2.getLeaves().get(0);
        assertEquals( 1, prj2.getColumn() );
        assertEquals( DataType.INTEGER, prj2.getResultType() );
        
        assertEquals( POGlobalRearrange.class, phyPlan.getSuccessors(localR).get(0).getClass() );
        POGlobalRearrange globalR = (POGlobalRearrange)phyPlan.getSuccessors(localR).get(0);
        assertEquals( DataType.TUPLE, globalR.getResultType() );

View Full Code Here

        POLocalRearrange localR = (POLocalRearrange)phyPlan.getSuccessors(load).get(0);
        assertEquals( 1, localR.getPlans().size() );
        PhysicalPlan cogroupPlan = localR.getPlans().get(0);
        assertEquals( 1, cogroupPlan.getLeaves().size() );        
        assertEquals( POProject.class, cogroupPlan.getLeaves().get(0).getClass() );
        POProject prj = (POProject)cogroupPlan.getLeaves().get(0);
        assertEquals( 0, prj.getColumn() );
        assertEquals( DataType.CHARARRAY, prj.getResultType() );
        
        assertEquals( POGlobalRearrange.class, phyPlan.getSuccessors(localR).get(0).getClass() );
        POGlobalRearrange globalR = (POGlobalRearrange)phyPlan.getSuccessors(localR).get(0);
        assertEquals( DataType.TUPLE, globalR.getResultType() );
        
        assertEquals( POLoad.class, phyPlan.getRoots().get(1).getClass() );
        POLoad load2 = (POLoad)phyPlan.getRoots().get(0);
        
        assertEquals( POLocalRearrange.class, phyPlan.getSuccessors(load2).get(0).getClass() );
        POLocalRearrange localR2 = (POLocalRearrange)phyPlan.getSuccessors(load2).get(0);
        assertEquals( 1, localR2.getPlans().size() );
        PhysicalPlan cogroupPlan2 = localR2.getPlans().get(0);
        POProject prj2 = (POProject)cogroupPlan2.getLeaves().get(0);
        assertEquals( 0, prj2.getColumn() );
        assertEquals( DataType.CHARARRAY, prj2.getResultType() );
        
        assertEquals( POPackage.class, phyPlan.getSuccessors(globalR).get(0).getClass() );
        POPackage pack = (POPackage)phyPlan.getSuccessors(globalR).get(0);
        assertEquals( DataType.TUPLE, pack.getResultType() );
    }

View Full Code Here

        POLocalRearrange localR = (POLocalRearrange)phyPlan.getSuccessors(load).get(0);
        assertEquals( 2, localR.getPlans().size() );
        PhysicalPlan cogroupPlan = localR.getPlans().get(0);
        assertEquals( 1, cogroupPlan.getLeaves().size() );        
        assertEquals( POProject.class, cogroupPlan.getLeaves().get(0).getClass() );
        POProject prj = (POProject)cogroupPlan.getLeaves().get(0);
        assertEquals( 0, prj.getColumn() );
        assertEquals( DataType.CHARARRAY, prj.getResultType() );
        
        PhysicalPlan cogroupPlan2 = localR.getPlans().get(1);
        assertEquals( 1, cogroupPlan2.getLeaves().size() );        
        assertEquals( POProject.class, cogroupPlan2.getLeaves().get(0).getClass() );
        POProject prj2 = (POProject)cogroupPlan2.getLeaves().get(0);
        assertEquals( 1, prj2.getColumn() );
        assertEquals( DataType.INTEGER, prj2.getResultType() );
        
        assertEquals( POGlobalRearrange.class, phyPlan.getSuccessors(localR).get(0).getClass() );
        POGlobalRearrange globalR = (POGlobalRearrange)phyPlan.getSuccessors(localR).get(0);
        assertEquals( DataType.TUPLE, globalR.getResultType() );
        
        assertEquals( POLoad.class, phyPlan.getRoots().get(1).getClass() );
        POLoad load2 = (POLoad)phyPlan.getRoots().get(0);
        
        assertEquals( POLocalRearrange.class, phyPlan.getSuccessors(load2).get(0).getClass() );
        
        POLocalRearrange localR3 = (POLocalRearrange)phyPlan.getSuccessors(load2).get(0);
        assertEquals( 2, localR3.getPlans().size() );
        PhysicalPlan cogroupPlan3 = localR3.getPlans().get(0);
        POProject prj3 = (POProject)cogroupPlan3.getLeaves().get(0);
        assertEquals( 0, prj3.getColumn() );
        assertEquals( DataType.CHARARRAY, prj3.getResultType() );
        
        PhysicalPlan cogroupPlan4 = localR3.getPlans().get(1);
        POProject prj4 = (POProject)cogroupPlan4.getLeaves().get(0);
        assertEquals( 1, prj4.getColumn() );
        assertEquals( DataType.INTEGER, prj4.getResultType() );
        
        assertEquals( POPackage.class, phyPlan.getSuccessors(globalR).get(0).getClass() );
        POPackage pack = (POPackage)phyPlan.getSuccessors(globalR).get(0);
        assertEquals( DataType.TUPLE, pack.getResultType() );
    }

View Full Code Here

        Assert.assertTrue(sortPlanFS==null);
        
        PhysicalPlan pp = Util.buildPhysicalPlanFromNewLP(lp, pc);
        POStore poStore = (POStore)pp.getLeaves().get(0);
        POSort poSort = (POSort)pp.getPredecessors(poStore).get(0);
        POProject poProject = (POProject)poSort.getSortPlans().get(0).getLeaves().get(0);
        Assert.assertTrue(poProject.getResultType()==DataType.TUPLE);
    }

View Full Code Here

                // Then we can use this information to strip off these columns
                // from the "Value" and in POPackage stitch the right "Value"
                // tuple back by getting these columns from the "key". The goal
                // is reduce the amount of the data sent to Hadoop in the map.
                if(leaf instanceof POProject) {
                    POProject project = (POProject) leaf;
                    if(project.isStar()) {
                        if(plans.size() == 1) {
                            // note that we have a project *
                            mProjectStar  = true;
                            // key will be a tuple in this case
                            isKeyTuple = true;
                        } else {
                            // TODO: currently "group by (*, somethingelse)" is NOT
                            // allowed. So we should never get here. But once it is
                            // allowed, we will need to handle it. For now just log
                            log.debug("Project * in group by not being optimized in key-value transfer");
                        }
                    } else {
                        try {
                            List<PhysicalOperator> preds = plan.getPredecessors(leaf);
                            if (preds==null || !(preds.get(0) instanceof POProject))
                                mProjectedColsMap.put(project.getColumn(), keyIndex);
                        } catch (ExecException e) {
                            int errCode = 2070;
                            String msg = "Problem in accessing column from project operator.";
                            throw new PlanException(msg, errCode, PigException.BUG);
                        }
                    }
                    if(project.getResultType() == DataType.TUPLE)
                        isKeyTuple = true;
                }
                keyIndex++;
            }
        }

View Full Code Here

                // Then we can use this information to strip off these columns
                // from the "Value" and in POPackage stitch the right "Value"
                // tuple back by getting these columns from the "key". The goal
                // is reduce the amount of the data sent to Hadoop in the map.
                if(leaf instanceof POProject) {
                    POProject project = (POProject) leaf;
                    if(project.isStar()) {
                        if(secondaryPlans.size() == 1) {
                            // note that we have a project *
                            mSecondaryProjectStar  = true;
                            // key will be a tuple in this case
                            isSecondaryKeyTuple = true;
                        } else {
                            // TODO: currently "group by (*, somethingelse)" is NOT
                            // allowed. So we should never get here. But once it is
                            // allowed, we will need to handle it. For now just log
                            log.debug("Project * in group by not being optimized in key-value transfer");
                        }
                    } else {
                        try {
                            List<PhysicalOperator> preds = plan.getPredecessors(leaf);
                            if (preds==null || !(preds.get(0) instanceof POProject))
                                mSecondaryProjectedColsMap.put(project.getColumn(), keyIndex);
                        } catch (ExecException e) {
                            int errCode = 2070;
                            String msg = "Problem in accessing column from project operator.";
                            throw new PlanException(msg, errCode, PigException.BUG);
                        }
                    }
                    if(project.getResultType() == DataType.TUPLE)
                        isSecondaryKeyTuple = true;
                }
                keyIndex++;
            }
        }

View Full Code Here

                // Then we can use this information to strip off these columns
                // from the "Value" and in POPostCombinerPackage stitch the right "Value"
                // tuple back by getting these columns from the "key". The goal
                // is reduce the amount of the data sent to Hadoop in the map.
                if(leaf instanceof POProject) {
                    POProject project = (POProject) leaf;
                    if(project.isStar()) {
                        int errCode = 2021;
                        String msg = "Internal error. Unexpected operator project(*) in local rearrange inner plan.";
                        throw new PlanException(msg, errCode, PigException.BUG);
                    } else {
                        try {
                            mProjectedColsMap.put(project.getColumn(), keyIndex);
                        } catch (ExecException e) {
                            int errCode = 2070;
                            String msg = "Problem in accessing column from project operator.";
                            throw new PlanException(msg, errCode, PigException.BUG);
                        }
                    }
                    if(project.getResultType() == DataType.TUPLE)
                        isKeyTuple = true;
                }
                keyIndex++;
            }
        }

View Full Code Here

        assertEquals( null, ce1.getValue() ); 
        assertEquals( ConstantExpression.class, filRoots.get(3).getClass() );
        ConstantExpression ce2 = (ConstantExpression) filRoots.get(3);
        assertEquals( null, ce2.getValue() );
        assertEquals( POProject.class, filRoots.get(0).getClass() );
        POProject prj1 = (POProject) filRoots.get(0);
        assertEquals( 3, prj1.getColumn() );
        assertEquals( POProject.class, filRoots.get(2).getClass() );
        POProject prj2 = (POProject) filRoots.get(2);
        assertEquals( 0, prj2.getColumn() );




        // Check Store Operator
        PhysicalOperator stor = (PhysicalOperator)physicalPlan.getSuccessors(fil).get(0);
        assertEquals( POStore.class, stor.getClass() );

View Full Code Here

        assertEquals( null, ce1.getValue() ); 
        assertEquals( ConstantExpression.class, filRoots.get(2).getClass() );
        ConstantExpression ce2 = (ConstantExpression) filRoots.get(2);
        assertEquals( null, ce2.getValue() );
        assertEquals( POProject.class, filRoots.get(1).getClass() );
        POProject prj8 = (POProject) filRoots.get(1);
        assertEquals( 5, prj8.getColumn() );
        assertEquals( POProject.class, filRoots.get(3).getClass() );
        POProject prj9 = (POProject) filRoots.get(3);
        assertEquals( 2, prj9.getColumn() );
        
        
        PhysicalOperator fil2 = (PhysicalOperator)phyPlan.getSuccessors(ForE2).get(0);
        assertEquals( POFilter.class, fil2.getClass() );
        
        PhysicalOperator LoR3 = (PhysicalOperator)phyPlan.getSuccessors(fil2).get(0);
        assertEquals( POLocalRearrange.class, LoR3.getClass() );
        POLocalRearrange Lor3 = (POLocalRearrange) LoR3;
        PhysicalOperator prj12 = Lor3.getPlans().get(0).getLeaves().get(0);
        assertEquals( POProject.class, prj12.getClass() );
        assertEquals(3, ((POProject)prj12).getColumn() );
        
        PhysicalPlan filPlan2 = ((POFilter)fil2).getPlan();
        List<PhysicalOperator> filRoots2 = filPlan2.getRoots();
        
        assertEquals( ConstantExpression.class, filRoots2.get(0).getClass() );
        ConstantExpression ce3 = (ConstantExpression) filRoots2.get(0);
        assertEquals( null, ce3.getValue() ); 
        assertEquals( ConstantExpression.class, filRoots2.get(2).getClass() );
        ConstantExpression ce4 = (ConstantExpression) filRoots2.get(2);
        assertEquals( null, ce4.getValue() );
        assertEquals( POProject.class, filRoots2.get(1).getClass() );
        POProject prj10 = (POProject) filRoots2.get(1);
        assertEquals( 3, prj10.getColumn() );
        assertEquals( POProject.class, filRoots2.get(3).getClass() );
        POProject prj11 = (POProject) filRoots2.get(3);
        assertEquals( 0, prj11.getColumn() );
        
        // Check Store Operator
        PhysicalOperator stor = (PhysicalOperator)phyPlan.getLeaves().get(0);
        assertEquals( stor, phyPlan.getSuccessors(fil).get(0));
        assertEquals( POStore.class, stor.getClass() );

View Full Code Here

        
        assertEquals(foreach.getInputPlans().size(), 2);
        
        PhysicalPlan inner = foreach.getInputPlans().get(0);
        assertEquals(inner.size(), 1);
        POProject prj = (POProject)inner.getRoots().get(0);
        assertEquals(prj.getColumn(), 0);
        assertEquals(prj.getInputs(), null);


        inner = foreach.getInputPlans().get(1);
        assertEquals(inner.size(), 1);
        prj = (POProject)inner.getRoots().get(0);
        assertEquals(prj.getColumn(), 1);
        assertEquals(prj.getInputs(), null);
        Boolean[] flat = foreach.getToBeFlattened().toArray(new Boolean[0]);
        assertFalse(flat[0]);
        assertFalse(flat[1]);
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.CombinerOptimizer

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler$LastInputStreamingOptimizer

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRUtil

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer

org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange

org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POLocalRearrangeTezFactory

org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler

org.apache.pig.backend.hadoop.executionengine.tez.util.TezCompilerUtil

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.