Package org.apache.pig.test.utils

Examples of org.apache.pig.test.utils.LogicalPlanTester

255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
    }

    @Test
    public void testSortOptimization1() throws Exception {
        // Sort on something other than the main key
        LogicalPlanTester planTester = new LogicalPlanTester();
        planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
        planTester.buildPlan("B = group A by $0;");
        planTester.buildPlan("C = foreach B { D = limit A 10; E = order D by $1; generate group, E;};");

        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();
View Full Code Here
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
    }

    @Test
    public void testSortOptimization2() throws Exception {
        // Sort on the prefix of the main key
        LogicalPlanTester planTester = new LogicalPlanTester();
        planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
        planTester.buildPlan("B = group A by $0;");
        planTester.buildPlan("C = foreach B { D = limit A 10; E = order D by $0; generate group, E;};");

        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();
View Full Code Here
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
    }

    @Test
    public void testSortOptimization3() throws Exception {
        // Sort on the main key prefix / non main key prefix mixed
        LogicalPlanTester planTester = new LogicalPlanTester();
        planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
        planTester.buildPlan("B = group A by $0;");
        planTester
                .buildPlan("C = foreach B { D = limit A 10; E = order D by $1; F = order E by $0; generate group, F;};");

        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();
View Full Code Here
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
    }

    @Test
    public void testSortOptimization4() throws Exception {
        // Sort on the main key again
        LogicalPlanTester planTester = new LogicalPlanTester();
        planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
        planTester.buildPlan("B = group A by $0;");
        planTester.buildPlan("C = foreach B { D = limit A 10; E = order D by $0, $1, $2; generate group, E;};");

        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();
View Full Code Here
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
    }

    @Test
    public void testSortOptimization5() throws Exception {
        // Sort on the two keys, we can only take off 1
        LogicalPlanTester planTester = new LogicalPlanTester();
        planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
        planTester.buildPlan("B = group A by $0;");
        planTester
                .buildPlan("C = foreach B { D = limit A 10; E = order D by $1; F = order E by $2; generate group, F;};");

        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();
View Full Code Here
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
    }

    @Test
    public void testSortOptimization6() throws Exception {
        // Sort desc
        LogicalPlanTester planTester = new LogicalPlanTester();
        planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
        planTester.buildPlan("B = group A by $0;");
        planTester.buildPlan("C = foreach B { D = order A by $0 desc; generate group, D;};");

        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();
View Full Code Here
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
    }

    @Test
    public void testSortOptimization7() throws Exception {
        // Sort asc on 1st key, desc on 2nd key
        LogicalPlanTester planTester = new LogicalPlanTester();
        planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
        planTester.buildPlan("B = group A by ($0, $1);");
        planTester.buildPlan("C = foreach B { D = order A by $0, $1 desc; generate group, D;};");

        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();
View Full Code Here
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
    // See PIG-1193
    @Test
    public void testSortOptimization8() throws Exception {
        // Sort desc, used in UDF twice
        LogicalPlanTester planTester = new LogicalPlanTester();
        planTester.buildPlan("A = LOAD 'input1' AS (a0);");
        planTester.buildPlan("B = group A all;");
        planTester.buildPlan("C = foreach B { D = order A by $0 desc; generate DIFF(D, D);};");

        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();
View Full Code Here
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
    public void testReducerNumEstimationForOrderBy() throws Exception{
       // use the estimation
        pc.getProperties().setProperty("pig.exec.reducers.bytes.per.reducer", "100");
        pc.getProperties().setProperty("pig.exec.reducers.max", "10");
       
        LogicalPlanTester planTester = new LogicalPlanTester(pc) ;
        Util.copyFromLocalToCluster(cluster, "test/org/apache/pig/test/data/passwd", "/passwd");
        planTester.buildPlan("a = load '/passwd';");
        LogicalPlan lp = planTester.buildPlan("b = order a by $0;");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        POStore store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);

        MROperPlan mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);
        assertEquals(2, mrPlan.size());    
       
        MapReduceOper sort = mrPlan.getLeaves().get(0);       
        long reducer=Math.min((long)Math.ceil(new File("test/org/apache/pig/test/data/passwd").length()/100.0), 10);
        assertEquals(reducer, sort.getRequestedParallelism());
       
        // use the PARALLEL key word, it will override the estimated reducer number
        planTester = new LogicalPlanTester(pc) ;
        planTester.buildPlan("a = load '/passwd';");
        lp = planTester.buildPlan("b = order a by $0 PARALLEL 2;");
        pp = Util.buildPhysicalPlan(lp, pc);
        store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
       
        mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);              
        assertEquals(2, mrPlan.size());    
       
        sort = mrPlan.getLeaves().get(0);       
        assertEquals(2, sort.getRequestedParallelism());
       
        // the estimation won't take effect when it apply to non-dfs or the files doesn't exist, such as hbase
        planTester = new LogicalPlanTester(pc) ;
        planTester.buildPlan("a = load 'hbase://passwd' using org.apache.pig.backend.hadoop.hbase.HBaseStorage('c:f1 c:f2');");
        lp = planTester.buildPlan("b = order a by $0 ;");
        pp = Util.buildPhysicalPlan(lp, pc);
        store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);              
        assertEquals(2, mrPlan.size());    
       
        sort = mrPlan.getLeaves().get(0);
       
        assertEquals(1, sort.getRequestedParallelism());
       
        // test order by with three jobs (after optimization)
        planTester = new LogicalPlanTester(pc) ;
        planTester.buildPlan("a = load '/passwd';");
        planTester.buildPlan("b = foreach a generate $0, $1, $2;");
        lp = planTester.buildPlan("c = order b by $0;");
        pp = Util.buildPhysicalPlan(lp, pc);
        store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
       
        mrPlan = Util.buildMRPlanWithOptimizer(pp, pc);
View Full Code Here
260
261
262
263
264
265
266
267
268
269
270
271
        }
    }
   
    @Test
    public void testNonDfsLocation() throws Exception {
        LogicalPlanTester lpt = new LogicalPlanTester();
        String nonDfsUrl = "har://hdfs-namenode/user/foo/";
        LogicalPlan lp = lpt.buildPlan("a = load '" + nonDfsUrl + "';");
        LOLoad load = (LOLoad) lp.getRoots().get(0);
        nonDfsUrl = nonDfsUrl.replaceFirst("/$", "");
        Assert.assertEquals(nonDfsUrl, load.getInputFile().getFileName());
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.test.utils.LogicalPlanTester

  • org.apache.pig.impl.logicalLayer.optimizer.LogicalOptimizer
  • org.apache.pig.impl.logicalLayer.optimizer.SchemaCalculator
  • org.apache.pig.impl.logicalLayer.optimizer.SchemaRemover
  • org.apache.pig.impl.logicalLayer.validators.TypeCheckingValidator
  • org.apache.pig.impl.PigContext
  • org.apache.pig.impl.plan.CompilationMessageCollector
  • org.apache.pig.test.TestCollectedGroup
  • org.apache.pig.test.TestCombiner
  • org.apache.pig.test.TestExperimentalColumnPrune
  • org.apache.pig.test.TestExperimentalLogToPhyTranslationVisitor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.