Examples of MROperPlan

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan
A Plan used to create the plan of Map Reduce Operators which can be converted into the Job Control object. This is necessary to capture the dependencies among jobs

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan

        planTester
                .buildPlan("C = foreach B { D = limit A 10; E = order D by $1; F = order E by $2; generate group, F;};");


        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);


        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();


        assertTrue(so.getNumMRUseSecondaryKey() == 1);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan

        planTester.buildPlan("B = group A by $0;");
        planTester.buildPlan("C = foreach B { D = order A by $0 desc; generate group, D;};");


        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);


        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();


        assertTrue(so.getNumMRUseSecondaryKey() == 1);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan

        planTester.buildPlan("B = group A by ($0, $1);");
        planTester.buildPlan("C = foreach B { D = order A by $0, $1 desc; generate group, D;};");


        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);


        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();


        assertTrue(so.getNumMRUseSecondaryKey() == 1);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan

        planTester.buildPlan("B = group A all;");
        planTester.buildPlan("C = foreach B { D = order A by $0 desc; generate DIFF(D, D);};");


        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);


        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();


        assertTrue(so.getNumMRUseSecondaryKey() == 1);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan

      planTester.buildPlan("b = order a by $0;");
      planTester.buildPlan("c = limit b 10;");
      LogicalPlan lp = planTester.buildPlan("store c into '/tmp';");
      
      PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
      MapReduceOper mrOper = mrPlan.getRoots().get(0);
      int count = 1;
      
      while(mrPlan.getSuccessors(mrOper) != null) {
        mrOper = mrPlan.getSuccessors(mrOper).get(0);
        ++count;
      }        
      assertTrue(count == 3);
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan

      planTesterMR.buildPlan("b = order a by $0 parallel 2;");
      planTesterMR.buildPlan("c = limit b 10;");
      LogicalPlan lp = planTesterMR.buildPlan("store c into '/tmp';");
      
      PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
      MapReduceOper mrOper = mrPlan.getRoots().get(0);
      int count = 1;
      
      while(mrPlan.getSuccessors(mrOper) != null) {
        mrOper = mrPlan.getSuccessors(mrOper).get(0);
        ++count;
      }        
      assertTrue(count == 4);
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan

        planTester.buildPlan("b = load 'input2';");
        planTester.buildPlan("c = join a by $0, b by $0;");
        LogicalPlan lp = planTester.buildPlan("store c into '/tmp';");
        
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
        MapReduceOper mrOper = mrPlan.getRoots().get(0);
        
        assertTrue(mrOper.UDFs.size()==2);
        assertTrue(mrOper.UDFs.size()==2);
        assertTrue(mrOper.UDFs.contains("BinStorage"));
        assertTrue(mrOper.UDFs.contains("org.apache.pig.builtin.PigStorage"));

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan

            TestMergeJoin.DummyIndexableLoader.class.getName() + ";");
        planTester.buildPlan("c = join a by $0, b by $0 using \"merge\";");
        LogicalPlan lp = planTester.buildPlan("store c into '/tmp';");
        
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mp = Util.buildMRPlan(pp, pc);
        assertEquals("Checking number of MR Jobs for merge join with " +
            "IndexableLoadFunc:", 1, mp.size());
        
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan

        planTester.buildPlan("e = foreach d generate a1+a2;");
        LogicalPlan lp = planTester.buildPlan("store e into '/tmp';");
        planTester.typeCheckPlan(lp);
        
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mp = Util.buildMRPlan(pp, pc);
        MapReduceOper op = mp.getLeaves().get(0);
        assertTrue(op.UDFs.contains(new FuncSpec(PigStorageNoDefCtor.class.getName())+"('\t')"));
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan

        String compiledPlan, goldenPlan = null;
        int MAX_SIZE = 100000;
        MRCompiler comp = new MRCompiler(pp, pc);
        comp.compile();


        MROperPlan mrp = comp.getMRPlan();
        PlanPrinter ppp = new PlanPrinter(mrp);
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        ppp.print(baos);
        compiledPlan = baos.toString();

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.