Examples of org.apache.pig.experimental.logical.relational.LogicalPlan

Package org.apache.pig.experimental.logical.relational

Examples of org.apache.pig.experimental.logical.relational.LogicalPlan

org.apache.pig.experimental.logical.relational.LogicalPlan
LogicalPlan is the logical view of relational operations Pig will execute for a given script. Note that it contains only relational operations. All expressions will be contained in LogicalExpressionPlans inside each relational operator.

        @SuppressWarnings("unchecked")
        private void addForeach(LogicalRelationalOperator op) throws IOException {            
            LOForEach foreach = new LOForEach(op.getPlan());
            
            // add foreach to the base plan
            LogicalPlan p = (LogicalPlan)op.getPlan();
            p.add(foreach);
            List<Operator> next = p.getSuccessors(op);           
            if (next != null) {
                Operator[] nextArray = next.toArray(new Operator[0]);
                for(Operator n: nextArray) {                  
                    Pair<Integer, Integer> pos = p.disconnect(op, n);           
                    p.connect(foreach, pos.first, n, pos.second);
                }
            }
            
            p.connect(op, foreach);                        
            
            LogicalPlan innerPlan = new LogicalPlan();
            foreach.setInnerPlan(innerPlan);
                      
            // get output columns
            Set<Integer> cols = (Set<Integer>)op.getAnnotation(REQUIREDCOLS);            
            
            // build foreach inner plan
            List<LogicalExpressionPlan> exps = new ArrayList<LogicalExpressionPlan>();              
            LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[cols.size()]);
            innerPlan.add(gen);
            
            LogicalSchema schema = op.getSchema();
            for (int i=0, j=0; i<schema.size(); i++) {   
                if (!cols.contains(i)) {
                    continue;
                }
                               
                LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, i);
                innerLoad.getProjection().setUid(foreach);                    
                innerPlan.add(innerLoad);          
                innerPlan.connect(innerLoad, gen);
                
                LogicalExpressionPlan exp = new LogicalExpressionPlan();
                ProjectExpression prj = new ProjectExpression(exp, schema.getField(i).type, j++, 0);
                prj.setUid(gen);
                exp.add(prj);

View Full Code Here


    @Override
    protected OperatorPlan buildPattern() {        
        // the pattern that this rule looks for
        // is filter operator
        LogicalPlan plan = new LogicalPlan();
        LogicalRelationalOperator op = new LOFilter(plan);
        plan.add(op);        
        
        return plan;
    }

View Full Code Here


    @Override
    protected OperatorPlan buildPattern() {        
        // the pattern that this rule looks for
        // is join -> filter
        LogicalPlan plan = new LogicalPlan();
        LogicalRelationalOperator op1 = new LOJoin(plan);
        LogicalRelationalOperator op2 = new LOFilter(plan);
        plan.add(op1);
        plan.add(op2);
        plan.connect(op1, op2);
        
        return plan;
    }

View Full Code Here

        // C = join A on x, B on a;
        // D = filter C by x = a and x = 0 and b = 1 and y = b;
        // store D into 'whatever';
        
        // A = load
        LogicalPlan lp = new LogicalPlan();
        {
          LogicalSchema aschema = new LogicalSchema();
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.INTEGER));
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.INTEGER));
          aschema.getField(0).uid = 1;
          aschema.getField(1).uid = 2;
          LOLoad A = new LOLoad(new FileSpec("bla", new FuncSpec("PigStorage", "\t")), aschema, lp);
          lp.add(A);
          
          // B = load
          LogicalSchema bschema = new LogicalSchema();
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.INTEGER));
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.INTEGER));
          bschema.getField(0).uid = 3;
          bschema.getField(1).uid = 4;
          LOLoad B = new LOLoad(null, bschema, lp);
          lp.add(B);
          
          // C = join
          LogicalSchema cschema = new LogicalSchema();
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.INTEGER));
          cschema.getField(0).uid = 1;
          cschema.getField(1).uid = 2;
          cschema.getField(2).uid = 3;
          cschema.getField(3).uid = 4;
          LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
          ProjectExpression x = new ProjectExpression(aprojplan, DataType.INTEGER, 0, 0);
          x.neverUseForRealSetUid(1);
          LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
          ProjectExpression y = new ProjectExpression(bprojplan, DataType.INTEGER, 1, 0);
          y.neverUseForRealSetUid(3);
          MultiMap<Integer, LogicalExpressionPlan> mm = 
              new MultiMap<Integer, LogicalExpressionPlan>();
          mm.put(0, aprojplan);
          mm.put(1, bprojplan);
          LOJoin C = new LOJoin(lp, mm, JOINTYPE.HASH, new boolean[] {true, true});
          C.neverUseForRealSetSchema(cschema);
          lp.add(C);
          lp.connect(A, C);
          lp.connect(B, C);
        
          // D = filter
          LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
          ProjectExpression fx = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 0);
          fx.neverUseForRealSetUid(1);
          ConstantExpression fc0 = new ConstantExpression(filterPlan, DataType.INTEGER, new Integer(0));
          EqualExpression eq1 = new EqualExpression(filterPlan, fx, fc0);
          ProjectExpression fanotherx = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 0);
          fanotherx.neverUseForRealSetUid(1);
          ProjectExpression fa = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 2);
          fa.neverUseForRealSetUid(3);
          EqualExpression eq2 = new EqualExpression(filterPlan, fanotherx, fa);
          AndExpression and1 = new AndExpression(filterPlan, eq1, eq2);
          ProjectExpression fb = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 3);
          fb.neverUseForRealSetUid(4);
          ConstantExpression fc1 = new ConstantExpression(filterPlan, DataType.INTEGER, new Integer(1));
          EqualExpression eq3 = new EqualExpression(filterPlan, fb, fc1);
          AndExpression and2 = new AndExpression(filterPlan, and1, eq3);
          ProjectExpression fanotherb = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 3);
          fanotherb.neverUseForRealSetUid(4);
          ProjectExpression fy = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 1);
          fy.neverUseForRealSetUid(2);
          EqualExpression eq4 = new EqualExpression(filterPlan, fy, fanotherb);
          new AndExpression(filterPlan, and2, eq4);
        
          LOFilter D = new LOFilter(lp, filterPlan);
          D.neverUseForRealSetSchema(cschema);
          // Connect D to B, since the transform has happened.
          lp.add(D);
          lp.connect(C, D);
        }
        
        LogicalPlanOptimizer optimizer = new LogicalPlanOptimizer(lp, 500);
        optimizer.optimize();
        
        LogicalPlan expected = new LogicalPlan();
        {
            // A = load
          LogicalSchema aschema = new LogicalSchema();
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.INTEGER));
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.INTEGER));
          aschema.getField(0).uid = 1;
          aschema.getField(1).uid = 2;
          LOLoad A = new LOLoad(new FileSpec("bla", new FuncSpec("PigStorage", "\t")), aschema, expected);
          expected.add(A);
          
          // DA = filter
          LogicalExpressionPlan DAfilterPlan = new LogicalExpressionPlan();
          ProjectExpression fx = new ProjectExpression(DAfilterPlan, DataType.INTEGER, 0, 0);
          fx.neverUseForRealSetUid(1);
          ConstantExpression fc0 = new ConstantExpression(DAfilterPlan, DataType.INTEGER, new Integer(0));
          new EqualExpression(DAfilterPlan, fx, fc0);
          
          LOFilter DA = new LOFilter(expected, DAfilterPlan);
          DA.neverUseForRealSetSchema(aschema);
          expected.add(DA);
          expected.connect(A, DA);
          
          // B = load
          LogicalSchema bschema = new LogicalSchema();
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.INTEGER));
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.INTEGER));
          bschema.getField(0).uid = 3;
          bschema.getField(1).uid = 4;
          LOLoad B = new LOLoad(null, bschema, expected);
          expected.add(B);
          
          // DB = filter
          LogicalExpressionPlan DBfilterPlan = new LogicalExpressionPlan();
          ProjectExpression fb = new ProjectExpression(DBfilterPlan, DataType.INTEGER, 0, 1);
          fb.neverUseForRealSetUid(4);
          ConstantExpression fc1 = new ConstantExpression(DBfilterPlan, DataType.INTEGER, new Integer(1));
          new EqualExpression(DBfilterPlan, fb, fc1);
          
          LOFilter DB = new LOFilter(expected, DBfilterPlan);
          DB.neverUseForRealSetSchema(bschema);
          expected.add(DB);
          expected.connect(B, DB);
          
          // C = join
          LogicalSchema cschema = new LogicalSchema();
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.INTEGER));
          cschema.getField(0).uid = 1;
          cschema.getField(1).uid = 2;
          cschema.getField(2).uid = 3;
          cschema.getField(3).uid = 4;
          LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
          ProjectExpression x = new ProjectExpression(aprojplan, DataType.INTEGER, 0, 0);
          x.neverUseForRealSetUid(1);
          LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
          ProjectExpression y = new ProjectExpression(bprojplan, DataType.INTEGER, 1, 0);
          y.neverUseForRealSetUid(3);
          MultiMap<Integer, LogicalExpressionPlan> mm = 
              new MultiMap<Integer, LogicalExpressionPlan>();
          mm.put(0, aprojplan);
          mm.put(1, bprojplan);
          LOJoin C = new LOJoin(expected, mm, JOINTYPE.HASH, new boolean[] {true, true});
          C.neverUseForRealSetSchema(cschema);
          expected.add(C);
          expected.connect(DA, C);
          expected.connect(DB, C);
          
          // D = filter
          LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
          ProjectExpression fanotherx = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 0);
          fanotherx.neverUseForRealSetUid(1);
          ProjectExpression fa = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 2);
          fa.neverUseForRealSetUid(3);
          EqualExpression eq2 = new EqualExpression(filterPlan, fanotherx, fa);
          ProjectExpression fanotherb = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 3);
          fanotherb.neverUseForRealSetUid(4);
          ProjectExpression fy = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 1);
          fy.neverUseForRealSetUid(2);
          EqualExpression eq4 = new EqualExpression(filterPlan, fy, fanotherb);
          new AndExpression(filterPlan, eq2, eq4);
          
          LOFilter D = new LOFilter(expected, filterPlan);
          D.neverUseForRealSetSchema(cschema);
          expected.add(D);
          expected.connect(C, D);
        }
        
        assertTrue( lp.isEqual(expected) );
        // assertEquals(lp, expected);
    }

View Full Code Here

        // no foreach
        LogicalPlanTester lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v2);");
        lpt.buildPlan("b = filter a by v1==NULL;");        
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store b into 'empty';");  
        LogicalPlan newLogicalPlan = migratePlan(plan);
               
        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v2);");
        lpt.buildPlan("b = filter a by v1==NULL;");        
        plan = lpt.buildPlan("store b into 'empty';");  
        LogicalPlan expected = migratePlan(plan);
        
        assertTrue(expected.isEqual(newLogicalPlan));
        
        // no schema
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt';");
        lpt.buildPlan("b = foreach a generate $0, $1;");
        plan = lpt.buildPlan("store b into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt';");
        lpt.buildPlan("b = foreach a generate $0, $1;");
        plan = lpt.buildPlan("store b into 'empty';");  
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
    }

View Full Code Here

        // only foreach
        LogicalPlanTester lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v2);");
        lpt.buildPlan("b = foreach a generate id;");        
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store b into 'empty';");  
        LogicalPlan newLogicalPlan = migratePlan(plan);
               
        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id);");
        lpt.buildPlan("b = foreach a generate id;");        
        plan = lpt.buildPlan("store b into 'empty';");  
        LogicalPlan expected = migratePlan(plan);
        
        assertTrue(expected.isEqual(newLogicalPlan));
        
        // with filter
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v5, v3, v4, v2);");
        lpt.buildPlan("b = filter a by v1 != NULL AND (v2+v3)<100;");
        lpt.buildPlan("c = foreach b generate id;");
        plan = lpt.buildPlan("store c into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v3, v2);");
        lpt.buildPlan("b = filter a by v1 != NULL AND (v2+v3)<100;");
        lpt.buildPlan("c = foreach b generate id;");
        plan = lpt.buildPlan("store c into 'empty';"); 
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
        
        // with 2 foreach
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v5, v3, v4, v2);");
        lpt.buildPlan("b = foreach a generate v2, v5, v4;");
        lpt.buildPlan("c = foreach b generate v5, v4;");
        plan = lpt.buildPlan("store c into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (v5, v4);");
        lpt.buildPlan("b = foreach a generate v5, v4;");
        lpt.buildPlan("c = foreach b generate v5, v4;");
        plan = lpt.buildPlan("store c into 'empty';"); 
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
        
        // with 2 foreach
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v5, v3, v4, v2);");
        lpt.buildPlan("b = foreach a generate id, v1, v5, v3, v4;");
        lpt.buildPlan("c = foreach b generate v5, v4;");
        plan = lpt.buildPlan("store c into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (v5, v4);");
        lpt.buildPlan("b = foreach a generate v5, v4;");
        lpt.buildPlan("c = foreach b generate v5, v4;");
        plan = lpt.buildPlan("store c into 'empty';"); 
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
        
        // with 2 foreach and filter in between
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v5, v3, v4, v2);");
        lpt.buildPlan("b = foreach a generate v2, v5, v4;");
        lpt.buildPlan("c = filter b by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5, v4;");
        plan = lpt.buildPlan("store d into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (v5, v4, v2);");
        lpt.buildPlan("b = foreach a generate v2, v5, v4;");
        lpt.buildPlan("c = filter b by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5, v4;");
        plan = lpt.buildPlan("store d into 'empty';");  
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
        
        // with 2 foreach after join
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v2, v3);");
        lpt.buildPlan("b = load 'c.txt' as (id, v4, v5, v6);");
        lpt.buildPlan("c = join a by id, b by id;");       
        lpt.buildPlan("d = foreach c generate a::id, v5, v3, v4;");
        plan = lpt.buildPlan("store d into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v3);");
        lpt.buildPlan("b = load 'c.txt' as (id, v4, v5);");
        lpt.buildPlan("c = join a by id, b by id;");       
        lpt.buildPlan("d = foreach c generate a::id, v5, v3, v4;");
        plan = lpt.buildPlan("store d into 'empty';");  
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
        
        // with BinStorage, insert foreach after load
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' using BinStorage() as (id, v1, v5, v3, v4, v2);");        
        lpt.buildPlan("c = filter a by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5, v4;");
        plan = lpt.buildPlan("store d into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' using BinStorage() as (id, v1, v5, v3, v4, v2);");
        lpt.buildPlan("b = foreach a generate v5, v4, v2;");
        lpt.buildPlan("c = filter b by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5, v4;");
        plan = lpt.buildPlan("store d into 'empty';");  
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
        
       // with BinStorage, not to insert foreach after load if there is already one
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' using BinStorage() as (id, v1, v5, v3, v4, v2);");    
        lpt.buildPlan("b = foreach a generate v5, v4, v2;");
        lpt.buildPlan("c = filter b by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5;");
        plan = lpt.buildPlan("store d into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' using BinStorage() as (id, v1, v5, v3, v4, v2);");
        lpt.buildPlan("b = foreach a generate v5, v2;");
        lpt.buildPlan("c = filter b by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5;");
        plan = lpt.buildPlan("store d into 'empty';");  
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
        
       // with BinStorage, not to insert foreach after load if there is already one
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' using BinStorage() as (id, v1, v5, v3, v4, v2);");    
        lpt.buildPlan("b = foreach a generate v5, v4, v2, 10;");
        lpt.buildPlan("c = filter b by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5;");
        plan = lpt.buildPlan("store d into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' using BinStorage() as (id, v1, v5, v3, v4, v2);");
        lpt.buildPlan("b = foreach a generate v5, v2, 10;");
        lpt.buildPlan("c = filter b by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5;");
        plan = lpt.buildPlan("store d into 'empty';");  
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
    }

View Full Code Here

         // only foreach
        LogicalPlanTester lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v1, m:map[]);");
        lpt.buildPlan("b = foreach a generate id, m#'path';");        
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store b into 'empty';");  
        LogicalPlan newLogicalPlan = migratePlan(plan);
               
        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, m:map[]);");
        lpt.buildPlan("b = foreach a generate id, m#'path';");        
        plan = lpt.buildPlan("store b into 'empty';");  
        LogicalPlan expected = migratePlan(plan);
        
        assertTrue(expected.isEqual(newLogicalPlan));
        
        LOLoad op = (LOLoad)newLogicalPlan.getSources().get(0);
        Map<Integer,Set<String>> annotation = 
                (Map<Integer, Set<String>>) op.getAnnotation(MapKeysPruneHelper.REQUIRED_MAPKEYS);
        assertEquals(annotation.size(), 1);
        Set<String> s = new HashSet<String>();
        s.add("path");
        assertEquals(annotation.get(2), s);
        
        // foreach with join
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v1, m:map[]);");
        lpt.buildPlan("b = load 'd.txt' as (id, v1, m:map[]);");
        lpt.buildPlan("c = join a by id, b by id;");
        lpt.buildPlan("d = filter c by a::m#'path' != NULL;");
        lpt.buildPlan("e = foreach d generate a::id, b::id, b::m#'path', a::m;");        
        plan = lpt.buildPlan("store e into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, m:map[]);");
        lpt.buildPlan("b = load 'd.txt' as (id, m:map[]);");
        lpt.buildPlan("c = join a by id, b by id;");
        lpt.buildPlan("d = filter c by a::m#'path' != NULL;");
        lpt.buildPlan("e = foreach d generate a::id, b::id, b::m#'path', a::m;");        
        plan = lpt.buildPlan("store e into 'empty';");  
        expected = migratePlan(plan);
        
        assertTrue(expected.isEqual(newLogicalPlan));
        
        List<Operator> ll = newLogicalPlan.getSources();
        assertEquals(ll.size(), 2);
        LOLoad loada = null;
        LOLoad loadb = null;

View Full Code Here

        lpt.buildPlan("a = load 'd.txt' as (id, v:bag{t:(s1,s2,s3)});");
        lpt.buildPlan("b = filter a by id>10;");
        lpt.buildPlan("c = foreach b generate id, FLATTEN(v);");    
        lpt.buildPlan("d = foreach c generate id, v::s2;");    
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store d into 'empty';");  
        LogicalPlan newLogicalPlan = migratePlan(plan);
               
        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v:bag{t:(s1,s2,s3)});");
        lpt.buildPlan("b = filter a by id>10;");
        lpt.buildPlan("c = foreach b generate id, FLATTEN(v);");    
        lpt.buildPlan("d = foreach c generate id, v::s2;");    
        plan = lpt.buildPlan("store d into 'empty';");
        LogicalPlan expected = migratePlan(plan);
        
        assertTrue(expected.isEqual(newLogicalPlan));
    }

View Full Code Here

        LogicalPlanTester lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v2);");
        lpt.buildPlan("b = filter a by v1>10;");
        lpt.buildPlan("c = foreach b generate id;");        
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store c into 'empty';");  
        LogicalPlan newLogicalPlan = migratePlan(plan);
               
        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v1);");
        lpt.buildPlan("b = filter a by v1>10;");
        lpt.buildPlan("c = foreach b generate id;");      
        plan = lpt.buildPlan("store c into 'empty';");  
        LogicalPlan expected = migratePlan(plan);
        
        assertTrue(expected.isEqual(newLogicalPlan));
        
        // join with foreach
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v2);");
        lpt.buildPlan("b = load 'd.txt' as (id, v1, v2);");
        lpt.buildPlan("c = join a by id, b by id;");
        lpt.buildPlan("d = filter c by a::v1>b::v1;");
        lpt.buildPlan("e = foreach d generate a::id;");        
        plan = lpt.buildPlan("store e into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester();
        lpt.buildPlan("a = load 'd.txt' as (id, v1);");
        lpt.buildPlan("b = load 'd.txt' as (id, v1);");
        lpt.buildPlan("c = join a by id, b by id;");
        lpt.buildPlan("d = foreach c generate a::id, a::v1, b::v1;");        
        lpt.buildPlan("e = filter d by a::v1>b::v1;");
        lpt.buildPlan("f = foreach e generate a::id;");        
        plan = lpt.buildPlan("store f into 'empty';");  
        expected = migratePlan(plan);
        
        assertTrue(expected.isEqual(newLogicalPlan));
    }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.pig.experimental.logical.relational.LogicalPlan

org.apache.pig.experimental.logical.rules.AddForEach$AddForEachTransformer

org.apache.pig.experimental.logical.rules.ColumnMapKeyPrune$ColumnMapKeyPruneTransformer$ColumnPruneVisitor

org.apache.pig.experimental.logical.rules.ColumnPruneHelper$ColumnDependencyVisitor

org.apache.pig.experimental.logical.rules.FilterAboveForeach

org.apache.pig.experimental.logical.rules.MergeFilter

org.apache.pig.experimental.logical.rules.PushUpFilter

org.apache.pig.experimental.logical.rules.SplitFilter

org.apache.pig.test.TestExperimentalColumnPrune

org.apache.pig.test.TestExperimentalFilterAboveForeach

org.apache.pig.test.TestExperimentalFilterRule

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.