Examples of MapReduceOper

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper
An operator model for a Map Reduce job. Acts as a host to the plans that will execute in map, reduce and optionally combine phases. These will be embedded in the MROperPlan in order to capture the dependencies amongst jobs.

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

     * @param jobMroMap the map that maps {@link Job}s to {@link MapReduceOper}s
     */
    public static void updateJobMroMap(Map<Job, MapReduceOper> jobMroMap) {
        PigStats ps = PigStats.get();
        for (Map.Entry<Job, MapReduceOper> entry : jobMroMap.entrySet()) {
            MapReduceOper mro = entry.getValue();
            ps.mapMROperToJob(mro, entry.getKey());
        }        
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

            lp = lpt.buildPlan("store C into 'out';");
            MROperPlan mrPlan = Util.buildMRPlan(Util.buildPhysicalPlan(lp, pc),pc);            
            assertEquals(2,mrPlan.size());


            Iterator<MapReduceOper> itr = mrPlan.iterator();
            MapReduceOper oper = itr.next();
            assertTrue(oper.reducePlan.isEmpty());
            assertFalse(oper.mapPlan.isEmpty());


            oper = itr.next();
            assertFalse(oper.reducePlan.isEmpty());

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

        this.returnCode = returnCode; 
    }
        
    @SuppressWarnings("deprecation")
    JobStats addJobStats(Job job) {
        MapReduceOper mro = null;
        JobID jobId = job.getAssignedJobID();
        if (jobId != null) {
            mro = jobMroMap.get(jobId.toString());
        } else {
            mro = jobMroMap.get(job.toString());

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

        
        // Get the sort job
        Iterator<MapReduceOper> iter = mrPlan.getKeys().values().iterator();
        int counter = 0;
        while (iter.hasNext()) {
            MapReduceOper op = iter.next();
            counter++;
            if (op.isGlobalSort()) {
                assertTrue(op.getRequestedParallelism()==100);
            }
        }
        assertEquals(3, counter);


        pc.defaultParallel = -1;

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

        
        // Get the skew join job
        Iterator<MapReduceOper> iter = mrPlan.getKeys().values().iterator();
        int counter = 0;
        while (iter.hasNext()) {
            MapReduceOper op = iter.next();
            counter++;
            if (op.isSkewedJoin()) {
                assertTrue(op.getRequestedParallelism()==100);
            }
        }
        assertEquals(3, counter);
        
        pc.defaultParallel = -1;

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

        pp.addAsLeaf(store);


        MROperPlan mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);
        assertEquals(2, mrPlan.size());     
        
        MapReduceOper sort = mrPlan.getLeaves().get(0);        
        long reducer=Math.min((long)Math.ceil(new File("test/org/apache/pig/test/data/passwd").length()/100.0), 10);
        assertEquals(reducer, sort.getRequestedParallelism());
        
        // use the PARALLEL key word, it will override the estimated reducer number
        planTester = new LogicalPlanTester(pc) ;
        planTester.buildPlan("a = load '/passwd';");
        lp = planTester.buildPlan("b = order a by $0 PARALLEL 2;");
        pp = Util.buildPhysicalPlan(lp, pc);
        store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        
        mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);               
        assertEquals(2, mrPlan.size());     
        
        sort = mrPlan.getLeaves().get(0);        
        assertEquals(2, sort.getRequestedParallelism());
        
        // the estimation won't take effect when it apply to non-dfs or the files doesn't exist, such as hbase
        planTester = new LogicalPlanTester(pc) ;
        planTester.buildPlan("a = load 'hbase://passwd' using org.apache.pig.backend.hadoop.hbase.HBaseStorage('c:f1 c:f2');");
        lp = planTester.buildPlan("b = order a by $0 ;");
        pp = Util.buildPhysicalPlan(lp, pc);
        store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
 
        mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);               
        assertEquals(2, mrPlan.size());     
        
        sort = mrPlan.getLeaves().get(0);
        
        assertEquals(1, sort.getRequestedParallelism());
        
        // test order by with three jobs (after optimization)
        planTester = new LogicalPlanTester(pc) ;
        planTester.buildPlan("a = load '/passwd';");
        planTester.buildPlan("b = foreach a generate $0, $1, $2;");
        lp = planTester.buildPlan("c = order b by $0;");
        pp = Util.buildPhysicalPlan(lp, pc);
        store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        
        mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);
        assertEquals(3, mrPlan.size());     
        
        sort = mrPlan.getLeaves().get(0);       
        assertEquals(reducer, sort.getRequestedParallelism());
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

      planTester.buildPlan("c = limit b 10;");
      LogicalPlan lp = planTester.buildPlan("store c into '/tmp';");
      
      PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
      MapReduceOper mrOper = mrPlan.getRoots().get(0);
      int count = 1;
      
      while(mrPlan.getSuccessors(mrOper) != null) {
        mrOper = mrPlan.getSuccessors(mrOper).get(0);
        ++count;

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

      planTesterMR.buildPlan("c = limit b 10;");
      LogicalPlan lp = planTesterMR.buildPlan("store c into '/tmp';");
      
      PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
      MapReduceOper mrOper = mrPlan.getRoots().get(0);
      int count = 1;
      
      while(mrPlan.getSuccessors(mrOper) != null) {
        mrOper = mrPlan.getSuccessors(mrOper).get(0);
        ++count;

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

        planTester.buildPlan("c = join a by $0, b by $0;");
        LogicalPlan lp = planTester.buildPlan("store c into '/tmp';");
        
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
        MapReduceOper mrOper = mrPlan.getRoots().get(0);
        
        assertTrue(mrOper.UDFs.size()==2);
        assertTrue(mrOper.UDFs.size()==2);
        assertTrue(mrOper.UDFs.contains("BinStorage"));
        assertTrue(mrOper.UDFs.contains("org.apache.pig.builtin.PigStorage"));

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

        LogicalPlan lp = planTester.buildPlan("store e into '/tmp';");
        planTester.typeCheckPlan(lp);
        
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mp = Util.buildMRPlan(pp, pc);
        MapReduceOper op = mp.getLeaves().get(0);
        assertTrue(op.UDFs.contains(new FuncSpec(PigStorageNoDefCtor.class.getName())+"('\t')"));
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.