Examples of SampleOptimizer

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SampleOptimizer
A visitor to optimize plans that have a sample job that immediately follows a load/store only MR job. These kinds of plans are generated for order bys, and will soon be generated for joins that need to sample their data first. These can be changed so that the RandomSampleLoader subsumes the loader used in the first job and then removes the first job.

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SampleOptimizer

        }
        
        // Before optimizer visits, number of MR jobs = 3.
        assertEquals(3,count);   


        SampleOptimizer so = new SampleOptimizer(mrPlan);
        so.visit();


        count = 1;
        mrOper = mrPlan.getRoots().get(0);
        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SampleOptimizer

            ++count;
        }        
        // Before optimizer visits, number of MR jobs = 3.
        assertEquals(3,count);


        SampleOptimizer so = new SampleOptimizer(mrPlan);
        so.visit();


        count = 1;
        mrOper = mrPlan.getRoots().get(0);
        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SampleOptimizer

            ++count;
        }        
        // Before optimizer visits, number of MR jobs = 3.
        assertEquals(3,count);


        SampleOptimizer so = new SampleOptimizer(mrPlan);
        so.visit();


        count = 1;
        mrOper = mrPlan.getRoots().get(0);
        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SampleOptimizer

        }
        
        // Before optimizer visits, number of MR jobs = 3.
        assertEquals(3,count);   


        SampleOptimizer so = new SampleOptimizer(mrPlan);
        so.visit();


        count = 1;
        mrOper = mrPlan.getRoots().get(0);
        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SampleOptimizer

            ++count;
        }        
        // Before optimizer visits, number of MR jobs = 3.
        assertEquals(3,count);


        SampleOptimizer so = new SampleOptimizer(mrPlan);
        so.visit();


        count = 1;
        mrOper = mrPlan.getRoots().get(0);
        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SampleOptimizer

            ++count;
        }        
        // Before optimizer visits, number of MR jobs = 3.
        assertEquals(3,count);


        SampleOptimizer so = new SampleOptimizer(mrPlan);
        so.visit();


        count = 1;
        mrOper = mrPlan.getRoots().get(0);
        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SampleOptimizer

        }
        
        // Before optimizer visits, number of MR jobs = 3.
        assertEquals(3,count);   


        SampleOptimizer so = new SampleOptimizer(mrPlan, pc);
        so.visit();


        count = 1;
        mrOper = mrPlan.getRoots().get(0);
        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SampleOptimizer

            ++count;
        }        
        // Before optimizer visits, number of MR jobs = 3.
        assertEquals(3,count);


        SampleOptimizer so = new SampleOptimizer(mrPlan, pc);
        so.visit();


        count = 1;
        mrOper = mrPlan.getRoots().get(0);
        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SampleOptimizer

            ++count;
        }        
        // Before optimizer visits, number of MR jobs = 3.
        assertEquals(3,count);


        SampleOptimizer so = new SampleOptimizer(mrPlan, pc);
        so.visit();


        count = 1;
        mrOper = mrPlan.getRoots().get(0);
        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SampleOptimizer

            ++count;
        }        
        // Before optimizer visits, number of MR jobs = 3.
        assertEquals(3,count);


        SampleOptimizer so = new SampleOptimizer(mrPlan, pc);
        so.visit();


        count = 1;
        mrOper = mrPlan.getRoots().get(0);
        // the first mrOper should be the sampling job - it's udf list should only
        // contain BinStorage

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.