Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LogicalPlan

62
63
64
65
66
67
68
69
70
71
72
73
            Operator op = ops.next();
            if (op instanceof LOSplit || op instanceof LOStore)
                continue;
            List<Operator> succs = plan.getSuccessors(op);
            if (succs != null && succs.size() >= 2) {
                OperatorPlan match = new LogicalPlan();
                match.add(op);
                ll.add(match);
            }
        }
        return ll;
    }
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
    @Override
    protected OperatorPlan buildPattern() {
        // the pattern that this rule looks for
        // is foreach -> filter
        LogicalPlan plan = new LogicalPlan();
        LogicalRelationalOperator foreach = new LOForEach(plan);
        LogicalRelationalOperator filter = new LOFilter(plan);
       
        plan.add(foreach);
        plan.add(filter);
        plan.connect(foreach, filter);

        return plan;
    }
View Full Code Here
50
51
52
53
54
55
56
57
58
59
        super( name, false );
    }

    @Override
    protected OperatorPlan buildPattern() {
        LogicalPlan plan = new LogicalPlan();
        LogicalRelationalOperator foreach = new LOForEach(plan);
        plan.add( foreach );
        return plan;
    }
View Full Code Here
988
989
990
991
992
993
994
995
996
997
        }
    }

    @Override
    protected OperatorPlan buildPattern() {
        LogicalPlan plan = new LogicalPlan();
        LogicalRelationalOperator op = new LOFilter(plan);
        plan.add(op);
        return plan;
    }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
    }

    @Override
    protected OperatorPlan buildPattern() {
        // match each foreach.
        LogicalPlan plan = new LogicalPlan();
        LogicalRelationalOperator load = new LOLoad (null, null, plan, null );
        plan.add( load );
//        LogicalRelationalOperator filter = new LOFilter( plan );
//        plan.add( filter );
//        plan.connect( load, filter );
        return plan;
    }
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
        }
        else {
            plan.connect(op, foreach);
        }
       
        LogicalPlan innerPlan = new LogicalPlan();
        foreach.setInnerPlan(innerPlan);
       
        LogicalSchema schema = op.getSchema();
       
        // build foreach inner plan
        List<LogicalExpressionPlan> exps = new ArrayList<LogicalExpressionPlan>();
        LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[schema.size()-columnsToDrop.size()]);
        innerPlan.add(gen);
       
        for (int i=0, j=0; i<schema.size(); i++) {
            if (columnsToDrop.contains(i)) {
                continue;
            }
           
            LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, i);
            innerPlan.add(innerLoad);
            innerPlan.connect(innerLoad, gen);
           
            LogicalExpressionPlan exp = new LogicalExpressionPlan();
            ProjectExpression prj = new ProjectExpression(exp, j++, -1, gen);
            exp.add(prj);
            exps.add(exp);
View Full Code Here
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
        planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
        planTester.buildPlan("B = foreach A generate $1, flatten($2);");       
        planTester.buildPlan("C = filter B by $0 < 18;");
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = planTester.buildPlan( "D = STORE C INTO 'empty';" );

        LogicalPlan newLogicalPlan = migrateAndOptimizePlan( plan );

        Operator load = newLogicalPlan.getSources().get( 0 );
        Assert.assertTrue( load instanceof LOLoad );
        Operator filter = newLogicalPlan.getSuccessors( load ).get( 0 );
        Assert.assertTrue( filter instanceof LOFilter );
        Operator fe = newLogicalPlan.getSuccessors( filter ).get( 0 );
        Assert.assertTrue( fe instanceof LOForEach );
        Operator store = newLogicalPlan.getSuccessors( fe ).get( 0 );
        Assert.assertTrue( store instanceof LOStore );
    }
View Full Code Here
407
408
409
410
411
412
413
414
415
416
417
        planTester.buildPlan("c = foreach b generate AVG(a.age) as age;");
        planTester.buildPlan("d = foreach a generate name, age;");
        planTester.buildPlan("e = filter d by age > c.age;");
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = planTester.buildPlan("f = store e into 'empty';");

        LogicalPlan newLogicalPlan = migrateAndOptimizePlan( plan );

        Operator store = newLogicalPlan.getSinks().get( 0 );
        Operator foreach = newLogicalPlan.getPredecessors(store).get(0);
        Assert.assertTrue( foreach instanceof LOForEach );
    }
View Full Code Here
415
416
417
418
419
420
421
422
423
424
        Operator foreach = newLogicalPlan.getPredecessors(store).get(0);
        Assert.assertTrue( foreach instanceof LOForEach );
    }

    private LogicalPlan migrateAndOptimizePlan(org.apache.pig.impl.logicalLayer.LogicalPlan plan) throws FrontendException {
        LogicalPlan newLogicalPlan = migratePlan( plan );
        PlanOptimizer optimizer = new MyPlanOptimizer( newLogicalPlan, 3 );
        optimizer.optimize();
        return newLogicalPlan;
    }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
        LogicalPlanTester lpt = new LogicalPlanTester( pc );
        lpt.buildPlan( "A = LOAD 'file.txt' AS (name, cuisines:bag{ t : ( cuisine ) } );" );
        lpt.buildPlan( "B = FOREACH A GENERATE name, flatten(cuisines);" );
        lpt.buildPlan( "C = FILTER B BY name == 'joe';" );
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan( "D = STORE C INTO 'empty';" ); 
        LogicalPlan newLogicalPlan = migrateAndOptimizePlan( plan );

        Operator load = newLogicalPlan.getSources().get( 0 );
        Assert.assertTrue( load instanceof LOLoad );
        Operator filter = newLogicalPlan.getSuccessors( load ).get( 0 );
        Assert.assertTrue( filter instanceof LOFilter );
        Operator fe1 = newLogicalPlan.getSuccessors( filter ).get( 0 );
        Assert.assertTrue( fe1 instanceof LOForEach );
        Operator fe2 = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
        Assert.assertTrue( fe2 instanceof LOForEach );
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.newplan.logical.relational.LogicalPlan

  • com.google.common.hash.HashFunction
  • org.apache.pig.backend.hadoop.executionengine.HExecutionEngine
  • org.apache.pig.impl.plan.CompilationMessageCollector
  • org.apache.pig.impl.util.HashOutputStream
  • org.apache.pig.newplan.logical.DotLOPrinter
  • org.apache.pig.newplan.logical.expression.ProjectExpression
  • org.apache.pig.newplan.logical.optimizer.DanglingNestedNodeRemover
  • org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer
  • org.apache.pig.newplan.logical.optimizer.LogicalPlanPrinter
  • org.apache.pig.newplan.logical.optimizer.SchemaResetter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.