Examples of MapReduceOper

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper
An operator model for a Map Reduce job. Acts as a host to the plans that will execute in map, reduce and optionally combine phases. These will be embedded in the MROperPlan in order to capture the dependencies amongst jobs.

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

      "c = limit b 10;" +
      "store c into 'output';";
      
      PhysicalPlan pp = Util.buildPp(pigServer, query);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
      MapReduceOper mrOper = mrPlan.getRoots().get(0);
      int count = 1;
      
      while(mrPlan.getSuccessors(mrOper) != null) {
        mrOper = mrPlan.getSuccessors(mrOper).get(0);
        ++count;

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

      "b = order a by $0 parallel 2;" +
      "c = limit b 10;" + "store c into 'output';";
      
      PhysicalPlan pp = Util.buildPp(pigServerMR, query);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
      MapReduceOper mrOper = mrPlan.getRoots().get(0);
      int count = 1;
      
      while(mrPlan.getSuccessors(mrOper) != null) {
        mrOper = mrPlan.getSuccessors(mrOper).get(0);
        ++count;

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

        "b = load 'input2';" +
        "c = join a by $0, b by $0;" + "store c into 'output';";
        
        PhysicalPlan pp = Util.buildPp(pigServer, query);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
        MapReduceOper mrOper = mrPlan.getRoots().get(0);
        
        assertTrue(mrOper.UDFs.size()==2);
        assertTrue(mrOper.UDFs.size()==2);
        assertTrue(mrOper.UDFs.contains("BinStorage"));
        assertTrue(mrOper.UDFs.contains("org.apache.pig.builtin.PigStorage"));

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

        "d = order c by a0;" +
        "e = foreach d generate a1+a2;" +
        "store e into 'output';";
        PhysicalPlan pp = Util.buildPp(pigServer, query);
        MROperPlan mp = Util.buildMRPlan(pp, pc);
        MapReduceOper op = mp.getLeaves().get(0);
        assertTrue(op.UDFs.contains(new FuncSpec(PigStorageNoDefCtor.class.getName())+"('\t')"));
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

        "c = limit b 7;" + "store c into 'output' using "
                + PigStorageNoDefCtor.class.getName() + "('\t');";
         
        PhysicalPlan pp = Util.buildPp(pigServerMR, query);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
        MapReduceOper mrOper = mrPlan.getRoots().get(0);
        int count = 1;
        
        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);
            ++count;
        }        
        assertTrue(count == 4);


        MapReduceOper op = mrPlan.getLeaves().get(0);
        assertTrue(op.UDFs.contains(new FuncSpec(PigStorageNoDefCtor.class.getName())+"('\t')"));
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

            "c = cogroup a by $0, b by $0 using 'merge';" +
            "store c into 'output';";
        
        PhysicalPlan pp = Util.buildPp(pigServer, query);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
        MapReduceOper mrOper = mrPlan.getRoots().get(0);
        
        assertTrue(mrOper.UDFs.contains(TestCollectableLoadFunc.class.getName()));
        mrOper = mrPlan.getSuccessors(mrOper).get(0);
        assertTrue(mrOper.UDFs.contains(TestCollectableLoadFunc.class.getName()));
        assertTrue(mrOper.UDFs.contains(TestIndexableLoadFunc.class.getName()));

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

            "c = join a by $0, b by $0 using 'merge';" +
            "store c into 'output';";
        
        PhysicalPlan pp = Util.buildPp(pigServer, query);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
        MapReduceOper mrOper = mrPlan.getRoots().get(0);
        
        assertTrue(mrOper.UDFs.contains(TestIndexableLoadFunc.class.getName()));
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

        LogicalPlan lp = planTester.buildPlan("store B into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);


        int count = 1;
        MapReduceOper mrOper = mrPlan.getRoots().get(0);
        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);
            ++count;
        }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

        LogicalPlan lp = planTester.buildPlan("store C into 'output';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);


        int count = 1;
        MapReduceOper mrOper = mrPlan.getRoots().get(0);
        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);
            ++count;
        }        
        // Before optimizer visits, number of MR jobs = 3.

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

        LogicalPlan lp = planTester.buildPlan("store C into 'output';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);


        int count = 1;
        MapReduceOper mrOper = mrPlan.getRoots().get(0);
        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);
            ++count;
        }        
        // Before optimizer visits, number of MR jobs = 3.

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.