Package org.apache.pig.backend.hadoop.executionengine.mapReduceLayer

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

60
61
62
63
64
65
66
67
68
69
70
        w.close();
       
        ScriptState ss = ScriptState.get();
        ss.setScript(new File("test.pig"));
        Configuration conf = new Configuration();
        MapReduceOper mro = new MapReduceOper(new OperatorKey());
        ss.addSettingsToConf(mro, conf);
       
        String s = conf.get("pig.script");
        String script = new String(Base64.decodeBase64(s.getBytes()));
       
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
        Util.createLocalInputFile( "testScript.py", script);
       
        ScriptState ss = ScriptState.get();
        ss.setScript(new File("testScript.py"));
        Configuration conf = new Configuration();
        MapReduceOper mro = new MapReduceOper(new OperatorKey());
        ss.addSettingsToConf(mro, conf);
       
        String s = conf.get("pig.script");
        String actual = new String(Base64.decodeBase64(s.getBytes()));
       
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
                    null);
            MROperPlan mp = getMRPlan(pp, pig.getPigContext());
           
            assertEquals(3, mp.getKeys().size());
           
            MapReduceOper mro = mp.getRoots().get(0);
            assertEquals("A,B,C", getAlias(mro));
           
            mro = mp.getSuccessors(mro).get(0);
            assertEquals("D", getAlias(mro));
            
View Full Code Here
520
521
522
523
524
525
526
527
528
529
530
531
532
533
       
        // Get the sort job
        Iterator<MapReduceOper> iter = mrPlan.getKeys().values().iterator();
        int counter = 0;
        while (iter.hasNext()) {
            MapReduceOper op = iter.next();
            counter++;
            if (op.isGlobalSort()) {
                assertTrue(op.getRequestedParallelism()==100);
            }
        }
        assertEquals(3, counter);

        pc.defaultParallel = -1;       
View Full Code Here
553
554
555
556
557
558
559
560
561
562
563
564
565
566
       
        // Get the skew join job
        Iterator<MapReduceOper> iter = mrPlan.getKeys().values().iterator();
        int counter = 0;
        while (iter.hasNext()) {
            MapReduceOper op = iter.next();
            counter++;
            if (op.isSkewedJoin()) {
                assertTrue(op.getRequestedParallelism()==100);
            }
        }
        assertEquals(3, counter);
       
        pc.defaultParallel = -1;       
View Full Code Here
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
        pp.addAsLeaf(store);

        MROperPlan mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);
        assertEquals(2, mrPlan.size());    
       
        MapReduceOper sort = mrPlan.getLeaves().get(0);       
        long reducer=Math.min((long)Math.ceil(new File("test/org/apache/pig/test/data/passwd").length()/100.0), 10);
        assertEquals(reducer, sort.getRequestedParallelism());
       
        // use the PARALLEL key word, it will override the estimated reducer number
        planTester = new LogicalPlanTester(pc) ;
        planTester.buildPlan("a = load '/passwd';");
        lp = planTester.buildPlan("b = order a by $0 PARALLEL 2;");
        pp = Util.buildPhysicalPlan(lp, pc);
        store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
       
        mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);              
        assertEquals(2, mrPlan.size());    
       
        sort = mrPlan.getLeaves().get(0);       
        assertEquals(2, sort.getRequestedParallelism());
       
        // the estimation won't take effect when it apply to non-dfs or the files doesn't exist, such as hbase
        planTester = new LogicalPlanTester(pc) ;
        planTester.buildPlan("a = load 'hbase://passwd' using org.apache.pig.backend.hadoop.hbase.HBaseStorage('c:f1 c:f2');");
        lp = planTester.buildPlan("b = order a by $0 ;");
        pp = Util.buildPhysicalPlan(lp, pc);
        store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);              
        assertEquals(2, mrPlan.size());    
       
        sort = mrPlan.getLeaves().get(0);
       
        assertEquals(1, sort.getRequestedParallelism());
       
        // test order by with three jobs (after optimization)
        planTester = new LogicalPlanTester(pc) ;
        planTester.buildPlan("a = load '/passwd';");
        planTester.buildPlan("b = foreach a generate $0, $1, $2;");
        lp = planTester.buildPlan("c = order b by $0;");
        pp = Util.buildPhysicalPlan(lp, pc);
        store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
       
        mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);
        assertEquals(3, mrPlan.size());    
       
        sort = mrPlan.getLeaves().get(0);      
        assertEquals(reducer, sort.getRequestedParallelism());
    }
View Full Code Here
876
877
878
879
880
881
882
883
884
885
886
      planTester.buildPlan("c = limit b 10;");
      LogicalPlan lp = planTester.buildPlan("store c into '/tmp';");
     
      PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
      MapReduceOper mrOper = mrPlan.getRoots().get(0);
      int count = 1;
     
      while(mrPlan.getSuccessors(mrOper) != null) {
        mrOper = mrPlan.getSuccessors(mrOper).get(0);
        ++count;
View Full Code Here
899
900
901
902
903
904
905
906
907
908
909
      planTesterMR.buildPlan("c = limit b 10;");
      LogicalPlan lp = planTesterMR.buildPlan("store c into '/tmp';");
     
      PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
      MapReduceOper mrOper = mrPlan.getRoots().get(0);
      int count = 1;
     
      while(mrPlan.getSuccessors(mrOper) != null) {
        mrOper = mrPlan.getSuccessors(mrOper).get(0);
        ++count;
View Full Code Here
918
919
920
921
922
923
924
925
926
927
928
        planTester.buildPlan("c = join a by $0, b by $0;");
        LogicalPlan lp = planTester.buildPlan("store c into '/tmp';");
       
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
        MapReduceOper mrOper = mrPlan.getRoots().get(0);
       
        assertTrue(mrOper.UDFs.size()==2);
        assertTrue(mrOper.UDFs.size()==2);
        assertTrue(mrOper.UDFs.contains("BinStorage"));
        assertTrue(mrOper.UDFs.contains("org.apache.pig.builtin.PigStorage"));
View Full Code Here
989
990
991
992
993
994
995
996
        LogicalPlan lp = planTester.buildPlan("store e into '/tmp';");
        planTester.typeCheckPlan(lp);
       
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mp = Util.buildMRPlan(pp, pc);
        MapReduceOper op = mp.getLeaves().get(0);
        assertTrue(op.UDFs.contains(new FuncSpec(PigStorageNoDefCtor.class.getName())+"('\t')"));
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

  • com.twitter.ambrose.pig.AmbrosePigProgressNotificationListener
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.POPackageAnnotator
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCounter
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion
  • org.apache.pig.impl.plan.OperatorKey
  • org.apache.pig.pen.LocalMapReduceSimulator
  • org.apache.pig.test.TestFRJoin2
  • org.apache.pig.test.TestJobControlCompiler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.