Package org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan

542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
        planTester.buildPlan("b = load 'input';");
        LogicalPlan lp = planTester.buildPlan("c = join a by $0, b by $0 using \"skewed\";");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        POStore store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        HExecutionEngine exe = pc.getExecutionEngine();
        ConfigurationValidator.validatePigProperties(exe.getConfiguration());
        Configuration conf = ConfigurationUtil.toConfiguration(exe.getConfiguration());
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);
       
        // Get the skew join job
        Iterator<MapReduceOper> iter = mrPlan.getKeys().values().iterator();
        int counter = 0;
        while (iter.hasNext()) {
            MapReduceOper op = iter.next();
            counter++;
            if (op.isSkewedJoin()) {
View Full Code Here
573
574
575
576
577
578
579
580
581
582
583
        planTester.buildPlan("a = load '/passwd';");
        LogicalPlan lp = planTester.buildPlan("b = group a by $0;");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        POStore store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
              
        pc.getConf().setProperty("pig.exec.reducers.bytes.per.reducer", "100");
        pc.getConf().setProperty("pig.exec.reducers.max", "10");
        HExecutionEngine exe = pc.getExecutionEngine();
        ConfigurationValidator.validatePigProperties(exe.getConfiguration());
View Full Code Here
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
        LogicalPlan lp = planTester.buildPlan("b = order a by $0;");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        POStore store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);

        MROperPlan mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);
        assertEquals(2, mrPlan.size());    
       
        MapReduceOper sort = mrPlan.getLeaves().get(0);       
        long reducer=Math.min((long)Math.ceil(new File("test/org/apache/pig/test/data/passwd").length()/100.0), 10);
        assertEquals(reducer, sort.getRequestedParallelism());
       
        // use the PARALLEL key word, it will override the estimated reducer number
        planTester = new LogicalPlanTester(pc) ;
        planTester.buildPlan("a = load '/passwd';");
        lp = planTester.buildPlan("b = order a by $0 PARALLEL 2;");
        pp = Util.buildPhysicalPlan(lp, pc);
        store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
       
        mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);              
        assertEquals(2, mrPlan.size());    
       
        sort = mrPlan.getLeaves().get(0);       
        assertEquals(2, sort.getRequestedParallelism());
       
        // the estimation won't take effect when it apply to non-dfs or the files doesn't exist, such as hbase
        planTester = new LogicalPlanTester(pc) ;
        planTester.buildPlan("a = load 'hbase://passwd' using org.apache.pig.backend.hadoop.hbase.HBaseStorage('c:f1 c:f2');");
        lp = planTester.buildPlan("b = order a by $0 ;");
        pp = Util.buildPhysicalPlan(lp, pc);
        store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);              
        assertEquals(2, mrPlan.size());    
       
        sort = mrPlan.getLeaves().get(0);
       
        assertEquals(1, sort.getRequestedParallelism());
       
        // test order by with three jobs (after optimization)
        planTester = new LogicalPlanTester(pc) ;
        planTester.buildPlan("a = load '/passwd';");
        planTester.buildPlan("b = foreach a generate $0, $1, $2;");
        lp = planTester.buildPlan("c = order b by $0;");
        pp = Util.buildPhysicalPlan(lp, pc);
        store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
       
        mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);
        assertEquals(3, mrPlan.size());    
       
        sort = mrPlan.getLeaves().get(0);      
        assertEquals(reducer, sort.getRequestedParallelism());
    }
View Full Code Here
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
      planTester.buildPlan("b = order a by $0;");
      planTester.buildPlan("c = limit b 10;");
      LogicalPlan lp = planTester.buildPlan("store c into '/tmp';");
     
      PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
      MapReduceOper mrOper = mrPlan.getRoots().get(0);
      int count = 1;
     
      while(mrPlan.getSuccessors(mrOper) != null) {
        mrOper = mrPlan.getSuccessors(mrOper).get(0);
        ++count;
      }       
      assertTrue(count == 3);
    }
View Full Code Here
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
      planTesterMR.buildPlan("b = order a by $0 parallel 2;");
      planTesterMR.buildPlan("c = limit b 10;");
      LogicalPlan lp = planTesterMR.buildPlan("store c into '/tmp';");
     
      PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
      MapReduceOper mrOper = mrPlan.getRoots().get(0);
      int count = 1;
     
      while(mrPlan.getSuccessors(mrOper) != null) {
        mrOper = mrPlan.getSuccessors(mrOper).get(0);
        ++count;
      }       
      assertTrue(count == 4);
    }
View Full Code Here
917
918
919
920
921
922
923
924
925
926
927
928
        planTester.buildPlan("b = load 'input2';");
        planTester.buildPlan("c = join a by $0, b by $0;");
        LogicalPlan lp = planTester.buildPlan("store c into '/tmp';");
       
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
        MapReduceOper mrOper = mrPlan.getRoots().get(0);
       
        assertTrue(mrOper.UDFs.size()==2);
        assertTrue(mrOper.UDFs.size()==2);
        assertTrue(mrOper.UDFs.contains("BinStorage"));
        assertTrue(mrOper.UDFs.contains("org.apache.pig.builtin.PigStorage"));
View Full Code Here
969
970
971
972
973
974
975
976
977
978
            TestMergeJoin.DummyIndexableLoader.class.getName() + ";");
        planTester.buildPlan("c = join a by $0, b by $0 using \"merge\";");
        LogicalPlan lp = planTester.buildPlan("store c into '/tmp';");
       
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mp = Util.buildMRPlan(pp, pc);
        assertEquals("Checking number of MR Jobs for merge join with " +
            "IndexableLoadFunc:", 1, mp.size());
       
    }
View Full Code Here
988
989
990
991
992
993
994
995
996
        planTester.buildPlan("e = foreach d generate a1+a2;");
        LogicalPlan lp = planTester.buildPlan("store e into '/tmp';");
        planTester.typeCheckPlan(lp);
       
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mp = Util.buildMRPlan(pp, pc);
        MapReduceOper op = mp.getLeaves().get(0);
        assertTrue(op.UDFs.contains(new FuncSpec(PigStorageNoDefCtor.class.getName())+"('\t')"));
    }
View Full Code Here
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
        String compiledPlan, goldenPlan = null;
        int MAX_SIZE = 100000;
        MRCompiler comp = new MRCompiler(pp, pc);
        comp.compile();

        MROperPlan mrp = comp.getMRPlan();
        PlanPrinter ppp = new PlanPrinter(mrp);
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        ppp.print(baos);
        compiledPlan = baos.toString();
View Full Code Here
199
200
201
202
203
204
205
206
207
208
209
            PigContext pigContext) throws MRCompilerException {
        super(plan, new DepthFirstWalker<PhysicalOperator, PhysicalPlan>(plan));
        this.plan = plan;
        this.pigContext = pigContext;
        splitsSeen = new HashMap<OperatorKey, MapReduceOper>();
        MRPlan = new MROperPlan();
        nig = NodeIdGenerator.getGenerator();
        r = new Random(1331);
        FileLocalizer.setR(r);
        udfFinder = new UDFFinder();
        List<PhysicalOperator> roots = plan.getRoots();
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan

  • com.netflix.lipstick.MRPlanCalculatorTest
  • com.netflix.lipstick.pigtolipstick.BasicP2LClientTest
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.LocalLauncher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.pen.LocalMapReduceSimulator
  • org.apache.pig.test.TestFRJoin2
  • org.apache.pig.test.TestGroupConstParallel
  • org.apache.pig.test.TestGroupConstParallelMR
  • org.apache.pig.test.TestJobControlCompiler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.