Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LogicalPlan


    }

    @Override
    protected OperatorPlan buildPattern() {
        // match each foreach.
        LogicalPlan plan = new LogicalPlan();
        LogicalRelationalOperator foreach1 = new LOForEach(plan);
        plan.add( foreach1 );
        return plan;
    }
View Full Code Here


           
            LOForEach foreach2 = (LOForEach)currentPlan.getSuccessors(foreach1).get(0);
            LOGenerate gen2 = (LOGenerate)foreach2.getInnerPlan().getSinks().get(0);
           
            LOForEach newForEach = new LOForEach(currentPlan);
            LogicalPlan newForEachInnerPlan = new LogicalPlan();
            newForEach.setInnerPlan(newForEachInnerPlan);
            newForEach.setAlias(foreach2.getAlias());
            newForEach.setRequestedParallelism(foreach1.getRequestedParallelisam());
            List<LogicalExpressionPlan> newExpList = new ArrayList<LogicalExpressionPlan>();
            LOGenerate newGen = new LOGenerate(newForEachInnerPlan, newExpList, gen2.getFlattenFlags());
            newGen.setUserDefinedSchema(gen2.getUserDefinedSchema());
            newForEachInnerPlan.add(newGen);
           
            for (LogicalExpressionPlan exp2 : gen2.getOutputPlans()) {
                LogicalExpressionPlan newExpPlan = new LogicalExpressionPlan();
                LogicalExpressionPlan exp2Copy = exp2.deepCopy();
                newExpPlan.merge(exp2Copy);
               
                // Add expression plan in 2nd ForEach
                List<Operator> exp2Sinks = new ArrayList<Operator>();
                exp2Sinks.addAll(newExpPlan.getSinks());
                for (Operator exp2Sink : exp2Sinks) {
                    if (exp2Sink instanceof ProjectExpression) {
                        // Find referred expression plan in 1st ForEach
                        ProjectExpression proj = (ProjectExpression)exp2Sink;
                        LOInnerLoad innerLoad = (LOInnerLoad)foreach2.getInnerPlan().getPredecessors(gen2).get(proj.getInputNum());
                        int exp1Pos = innerLoad.getProjection().getColNum();
                        LogicalExpressionPlan exp1 = gen1.getOutputPlans().get(exp1Pos);
                        LogicalExpressionPlan exp1Copy = exp1.deepCopy();
                        List<Operator> exp1Sources = newExpPlan.merge(exp1Copy);
                       
                        // Copy expression plan to the new ForEach, connect to the expression plan of 2nd ForEach
                        Operator exp1Source = exp1Sources.get(0);
                        if (newExpPlan.getPredecessors(exp2Sink)!=null) {
                            Operator exp2NextToSink = newExpPlan.getPredecessors(exp2Sink).get(0);
                            Pair<Integer, Integer> pos = newExpPlan.disconnect(exp2NextToSink, exp2Sink);
                            newExpPlan.remove(exp2Sink);
                            newExpPlan.connect(exp2NextToSink, pos.first, exp1Source, 0);
                        }
                        else {
                            newExpPlan.remove(exp2Sink);
                        }
                    }
                }
               
                // Copy referred ForEach1 inner plan to new ForEach
                List<Operator> exp1Sinks = newExpPlan.getSinks();
                for (Operator exp1Sink : exp1Sinks) {
                    if (exp1Sink instanceof ProjectExpression) {
                        Operator opNextToGen = addBranchToPlan(gen1, ((ProjectExpression)exp1Sink).getInputNum(), newForEachInnerPlan, newForEach);
                        newForEachInnerPlan.connect(opNextToGen, newGen);
                        int input = newForEachInnerPlan.getPredecessors(newGen).indexOf(opNextToGen);
                        ((ProjectExpression)exp1Sink).setInputNum(input);
                    }
                }
               
                newExpList.add(newExpPlan);
View Full Code Here

            "('srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int', " +
            "'srcid,mrkt') as (f1, f2, f3, f4, f5);");
        org.apache.pig.impl.logicalLayer.LogicalPlan lp = lpTester.buildPlan("b = filter a by " +
            "(f5 >= 20 and f2 == 'us') and (f1 == 10 and f3 == 15);");
       
        LogicalPlan newLogicalPlan = migrateAndOptimizePlan( lp );
       
        Assert.assertEquals("checking partition filter:",            
                    "((mrkt == 'us') and (srcid == 10))",
                    TestLoader.partFilter.toString());
        LOFilter filter = (LOFilter)newLogicalPlan.getSinks().get(0);
        String actual = PColFilterExtractor.getExpression(
                (LogicalExpression)filter.getFilterPlan().getSources().get(0)).
                toString().toLowerCase();
        Assert.assertEquals("checking trimmed filter expression:",
                "((f5 >= 20) and (f3 == 15))", actual);
View Full Code Here

        Assert.assertEquals("checking trimmed filter expression:",
                "((f5 >= 20) and (f3 == 15))", actual);
    }
   
    private LogicalPlan migrateAndOptimizePlan(org.apache.pig.impl.logicalLayer.LogicalPlan plan) throws IOException {
        LogicalPlan newLogicalPlan = migratePlan( plan );
        PlanOptimizer optimizer = new MyPlanOptimizer( newLogicalPlan, 3 );
        optimizer.optimize();
        return newLogicalPlan;
    }
View Full Code Here

            "('srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int', " +
            "'srcid') as (f1, f2, f3, f4, f5);");
        org.apache.pig.impl.logicalLayer.LogicalPlan lp = lpTester.buildPlan("b = filter a by " +
                "f5 >= 20 and f2 == 'us' and f3 == 15;");

        LogicalPlan newLogicalPlan = migrateAndOptimizePlan( lp );

        Assert.assertEquals("checking partition filter:",            
                    null,
                    TestLoader.partFilter);
        LOFilter filter = (LOFilter) newLogicalPlan.getSinks().get(0);
        String actual = PColFilterExtractor.getExpression(
                (LogicalExpression) filter.getFilterPlan().
                getSources().get(0)).
                toString().toLowerCase();
        Assert.assertEquals("checking trimmed filter expression:",
View Full Code Here

            "('srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int', " +
            "'srcid,mrkt,dstid,age') as (f1, f2, f3, f4, f5);");
        org.apache.pig.impl.logicalLayer.LogicalPlan lp = lpTester.buildPlan("b = filter a by " +
                "(f5 >= 20 or f2 == 'us') and (f1 == 10 and f3 == 15);");

        LogicalPlan newLogicalPlan = migrateAndOptimizePlan( lp );

        Assert.assertEquals("checking partition filter:",            
                    "(((age >= 20) or (mrkt == 'us')) and ((srcid == 10) and " +
                    "(dstid == 15)))",
                    TestLoader.partFilter.toString());
        Iterator<Operator> it = newLogicalPlan.getOperators();
        Assert.assertTrue("Checking that filter has been removed since it contained" +
            " only conditions on partition cols:",
            (it.next() instanceof LOLoad));
        Assert.assertFalse("Checking that filter has been removed since it contained" +
                " only conditions on partition cols:",
View Full Code Here

            "('srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int', " +
            "'srcid,mrkt') as (f1, f2, f3);");
        org.apache.pig.impl.logicalLayer.LogicalPlan lp = lpTester.buildPlan("b = filter a by " +
                "(age >= 20 and f2 == 'us') and (f1 == 10 and f3 == 15);");

        LogicalPlan newLogicalPlan = migrateAndOptimizePlan( lp );
       
        Assert.assertEquals("checking partition filter:",            
                    "((mrkt == 'us') and (srcid == 10))",
                    TestLoader.partFilter.toString());
        LOFilter filter = (LOFilter) newLogicalPlan.getSinks().get(0);
        String actual = PColFilterExtractor.getExpression(
                (LogicalExpression) filter.getFilterPlan().getSources().get(0)).
                toString().toLowerCase();
        Assert.assertEquals("checking trimmed filter expression:",
                "((age >= 20) and (f3 == 15))", actual);
View Full Code Here

                .buildPlan("d = foreach c generate $4 as bcookie:chararray, " +
                    "$5 as dstid:int, $0 as mrkt:chararray;");
       
        new PlanSetter(lp).visit();
       
        LogicalPlan newLogicalPlan = migrateAndOptimizePlan( lp );
       
        String partFilter = TestLoader.partFilter.toString();
        Assert.assertTrue( "(srcid == 20)".equals( partFilter ) ||  "(srcid == 10)".equals( partFilter ) );
       
        int counter = 0;
        Iterator<Operator> iter = newLogicalPlan.getOperators();
        while (iter.hasNext()) {
           Assert.assertTrue(!(iter.next() instanceof LOFilter));
            counter++;
        }     
        Assert.assertEquals(counter, 4);
View Full Code Here

    //// helper methods ///////
   
    private PColFilterExtractor test(org.apache.pig.impl.logicalLayer.LogicalPlan lp, List<String> partitionCols,
            String expPartFilterString, String expFilterString)
    throws IOException {
      LogicalPlan newLogicalPlan = migratePlan( lp );
        LOFilter filter = (LOFilter)newLogicalPlan.getSinks().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getFilterPlan(), partitionCols);
        pColExtractor.visit();
       
        if(expPartFilterString == null) {
View Full Code Here

        return pColExtractor;
    }
   
    private void negativeTest(org.apache.pig.impl.logicalLayer.LogicalPlan lp, List<String> partitionCols,
            int expectedErrorCode) throws VisitorException {
      LogicalPlan newLogicalPlan = migratePlan( lp );
        LOFilter filter = (LOFilter)newLogicalPlan.getSinks().get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getFilterPlan(), partitionCols);
        try {
            pColExtractor.visit();
        } catch(Exception e) {
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.logical.relational.LogicalPlan

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.