Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.LOForEach

1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
    @Test
    public void testQueryForeachGenerateStarWithSchema2() {
        String query = "foreach (load 'a' as (url, hitCount)) generate *, url ;";
        LogicalPlan lp = planTester.buildPlan(query);

        LOForEach foreach = (LOForEach)lp.getLeaves().get(0);
        RequiredFields foreachRelevantFields0 = foreach.getRelevantInputs(0, 0).get(0);
        assertTrue(foreachRelevantFields0.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields0.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields0.getFields().size() == 1);
        assertTrue(foreachRelevantFields0.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields0.getFields().get(0).second == 0);
       
        RequiredFields foreachRelevantFields1 = foreach.getRelevantInputs(0, 1).get(0);
        assertTrue(foreachRelevantFields1.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields1.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields1.getFields().size() == 1);
        assertTrue(foreachRelevantFields1.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields1.getFields().get(0).second == 1);
       
        RequiredFields foreachRelevantFields2 = foreach.getRelevantInputs(0, 2).get(0);
        assertTrue(foreachRelevantFields2.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields2.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields2.getFields().size() == 1);
        assertTrue(foreachRelevantFields2.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields2.getFields().get(0).second == 0);
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
    @Test
    public void testQueryForeach1() {
        String query = "foreach (load 'a') generate $1,$2;";
        LogicalPlan lp = planTester.buildPlan(query);
       
        LOForEach foreach = (LOForEach)lp.getLeaves().get(0);
        RequiredFields foreachRelevantFields0 = foreach.getRelevantInputs(0, 0).get(0);
        assertTrue(foreachRelevantFields0.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields0.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields0.getFields().size() == 1);
        assertTrue(foreachRelevantFields0.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields0.getFields().get(0).second == 1);
        RequiredFields foreachRelevantFields1 = foreach.getRelevantInputs(0, 1).get(0);
        assertTrue(foreachRelevantFields1.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields1.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields1.getFields().size() == 1);
        assertTrue(foreachRelevantFields1.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields1.getFields().get(0).second == 2);
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
    @Test
    public void testQueryForeach2() {
        String query = "foreach (load 'a') generate $1,$2+$3;";
        LogicalPlan lp = planTester.buildPlan(query);
       
        LOForEach foreach = (LOForEach)lp.getLeaves().get(0);
        RequiredFields foreachRelevantFields0 = foreach.getRelevantInputs(0, 0).get(0);
        assertTrue(foreachRelevantFields0.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields0.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields0.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields0.getFields().get(0).second == 1);
        RequiredFields foreachRelevantFields1 = foreach.getRelevantInputs(0, 1).get(0);
        assertTrue(foreachRelevantFields1.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields1.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields1.getFields().size() == 2);
        assertTrue(foreachRelevantFields1.getFields().contains(new Pair<Integer, Integer>(0, 2)));
        assertTrue(foreachRelevantFields1.getFields().contains(new Pair<Integer, Integer>(0, 3)));
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
    @Test
    public void testQueryForeach3() {
        String query = "foreach (load 'a') generate $1,CONCAT($2,$3);";
        LogicalPlan lp = planTester.buildPlan(query);
       
        LOForEach foreach = (LOForEach)lp.getLeaves().get(0);
        RequiredFields foreachRelevantFields0 = foreach.getRelevantInputs(0, 0).get(0);
        assertTrue(foreachRelevantFields0.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields0.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields0.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields0.getFields().get(0).second == 1);
        RequiredFields foreachRelevantFields1 = foreach.getRelevantInputs(0, 1).get(0);
        assertTrue(foreachRelevantFields1.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields1.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields1.getFields().size() == 2);
        assertTrue(foreachRelevantFields1.getFields().contains(new Pair<Integer, Integer>(0, 2)));
        assertTrue(foreachRelevantFields1.getFields().contains(new Pair<Integer, Integer>(0, 3)));
View Full Code Here
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
    public void testQueryForeach4() throws FrontendException {
        planTester.buildPlan("A = load 'a';");
        planTester.buildPlan("B = load 'b';");
        LogicalPlan lp = planTester.buildPlan("foreach (cogroup A by ($1), B by ($1)) generate A, flatten(B.($1, $2, $3));");
       
        LOForEach foreach = (LOForEach)lp.getLeaves().get(0);
        RequiredFields foreachRelevantFields0 = foreach.getRelevantInputs(0, 0).get(0);
        assertTrue(foreachRelevantFields0.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields0.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields0.getFields().size() == 1);
        assertTrue(foreachRelevantFields0.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields0.getFields().get(0).second == 1);
        RequiredFields foreachRelevantFields1 = foreach.getRelevantInputs(0, 1).get(0);
        assertTrue(foreachRelevantFields1.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields1.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields1.getFields().size() == 1);
        assertTrue(foreachRelevantFields1.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields1.getFields().get(0).second == 2);
        RequiredFields foreachRelevantFields2 = foreach.getRelevantInputs(0, 2).get(0);
        assertTrue(foreachRelevantFields2.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields2.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields2.getFields().size() == 1);
        assertTrue(foreachRelevantFields2.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields2.getFields().get(0).second == 2);
        RequiredFields foreachRelevantFields3 = foreach.getRelevantInputs(0, 3).get(0);
        assertTrue(foreachRelevantFields3.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields3.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields3.getFields().size() == 1);
        assertTrue(foreachRelevantFields3.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields3.getFields().get(0).second == 2);
        assertTrue(foreach.getRelevantInputs(0, 4) == null);
    }
View Full Code Here
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
                "B = order B by $0; " +
                "generate FLATTEN(A), B.($1, $2, $3) ;" +
                "};" ;
        LogicalPlan lp = planTester.buildPlan(query);

        LOForEach foreach = (LOForEach)lp.getLeaves().get(0);
        RequiredFields foreachRelevantFields0 = foreach.getRelevantInputs(0, 0).get(0);
        assertTrue(foreachRelevantFields0.getNeedAllFields()==false);
        assertTrue(foreachRelevantFields0.getNeedNoFields()==false);
        assertTrue(foreachRelevantFields0.getFields().size()==1);
        assertTrue(foreachRelevantFields0.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields0.getFields().get(0).second == 1);
       
        RequiredFields foreachRelevantFields1 = foreach.getRelevantInputs(0, 1).get(0);
        assertTrue(foreachRelevantFields1.getNeedAllFields()==false);
        assertTrue(foreachRelevantFields1.getNeedNoFields()==false);
        assertTrue(foreachRelevantFields1.getFields().size()==1);
        assertTrue(foreachRelevantFields1.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields1.getFields().get(0).second == 1);
       
        RequiredFields foreachRelevantFields2 = foreach.getRelevantInputs(0, 2).get(0);
        assertTrue(foreachRelevantFields2.getNeedAllFields()==false);
        assertTrue(foreachRelevantFields2.getNeedNoFields()==false);
        assertTrue(foreachRelevantFields2.getFields().size()==1);
        assertTrue(foreachRelevantFields2.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields2.getFields().get(0).second == 1);
       
        RequiredFields foreachRelevantFields3 = foreach.getRelevantInputs(0, 3).get(0);
        assertTrue(foreachRelevantFields3.getNeedAllFields()==false);
        assertTrue(foreachRelevantFields3.getNeedNoFields()==false);
        assertTrue(foreachRelevantFields3.getFields().size()==1);
        assertTrue(foreachRelevantFields3.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields3.getFields().get(0).second == 2);
View Full Code Here
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
        assertTrue(cogroupRelevantFields2.getNeedAllFields() == true);
        assertTrue(cogroupRelevantFields2.getNeedNoFields() == false);
        assertTrue(cogroupRelevantFields2.getFields() == null);
        assertTrue(cogroup.getRelevantInputs(0, 2).get(0) == null);
       
        LOForEach foreach = (LOForEach)lp.getLeaves().get(0);
        RequiredFields foreachRelevantFields0 = foreach.getRelevantInputs(0, 0).get(0);
        assertTrue(foreachRelevantFields0.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields0.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields0.getFields().size() == 1);
        assertTrue(foreachRelevantFields0.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields0.getFields().get(0).second == 0);
       
        RequiredFields foreachRelevantFields1 = foreach.getRelevantInputs(0, 1).get(0);
        assertTrue(foreachRelevantFields1.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields1.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields1.getFields().size() == 1);
        assertTrue(foreachRelevantFields1.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields1.getFields().get(0).second == 1);
       
        RequiredFields foreachRelevantFields2 = foreach.getRelevantInputs(0, 2).get(0);
        assertTrue(foreachRelevantFields2.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields2.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields2.getFields().size() == 1);
        assertTrue(foreachRelevantFields2.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields2.getFields().get(0).second == 1);
       
        RequiredFields foreachRelevantFields3 = foreach.getRelevantInputs(0, 3).get(0);
        assertTrue(foreachRelevantFields3.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields3.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields3.getFields().size() == 1);
        assertTrue(foreachRelevantFields3.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields3.getFields().get(0).second == 2);
       
        RequiredFields foreachRelevantFields4 = foreach.getRelevantInputs(0, 4).get(0);
        assertTrue(foreachRelevantFields4.getNeedAllFields() == false);
        assertTrue(foreachRelevantFields4.getNeedNoFields() == false);
        assertTrue(foreachRelevantFields4.getFields().size() == 1);
        assertTrue(foreachRelevantFields4.getFields().get(0).first == 0);
        assertTrue(foreachRelevantFields4.getFields().get(0).second == 2);
       
        assertTrue(foreach.getRelevantInputs(0, 5) == null);
    }
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
154
              return;
            }
            // Limit can be pushed in front of ForEach if it does not have a flatten
            else if (predecessor instanceof LOForEach)
            {
              LOForEach loForEach = (LOForEach)predecessor;
              List<Boolean> mFlatten = loForEach.getFlatten();
              boolean hasFlatten = false;
              for (Boolean b:mFlatten)
                if (b.equals(true)) hasFlatten = true;
             
              // We can safely move LOLimit up
View Full Code Here
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
                mPushBeforeInput = grandParentIndexes.iterator().next();
                return true;

            } else if (predecessor instanceof LOForEach) {

                LOForEach loForEach = (LOForEach) predecessor;
                List<Boolean> mFlatten = loForEach.getFlatten();
                boolean hasFlatten = false;
                for (Boolean b : mFlatten) {
                    if (b.equals(true)) {
                        hasFlatten = true;
                    }
                }

                // TODO
                // A better check is to examine each column in the filter's
                // required fields. If the column is the result of a flatten
                // then
                // return false else return true

                // for now if the foreach has a flatten then return false
                if (hasFlatten) {
                    return false;
                }

                Pair<Boolean, Set<Integer>> mappingResult = isRequiredFieldMapped(requiredField, predecessor.getProjectionMap());
                boolean mapped = mappingResult.first;
               
                // Check if it is a direct mapping, that is, project optionally followed by cast, so if project->project, it is not
                // considered as a mapping
                for (Pair<Integer, Integer> pair : requiredField.getFields())
                {
                    if (!isFieldSimple(loForEach.getForEachPlans().get(pair.second)))
                    {
                        mapped = false;
                        break;
                    }
                }
View Full Code Here
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
                log.warn("fieldsToRead on "+load+" throw an exception, skip it");
            }
        }
       
        // Loader does not support column pruning, insert foreach
        LOForEach forEach = null;
        if (response==null || !response.getRequiredFieldResponse())
        {
            Set<Integer> columnsToProject = new TreeSet<Integer>();
            for (LoadFunc.RequiredField rf : requiredFieldList.getFields())
                columnsToProject.add(rf.getIndex());
           
            ArrayList<Boolean> flattenList = new ArrayList<Boolean>();
            ArrayList<LogicalPlan> generatePlans = new ArrayList<LogicalPlan>();
            String scope = load.getOperatorKey().scope;
            for (int pos : columnsToProject) {
                LogicalPlan projectPlan = new LogicalPlan();
                LogicalOperator projectInput = load;
                ExpressionOperator column = new LOProject(projectPlan, new OperatorKey(scope, NodeIdGenerator.getGenerator().getNextNodeId(scope)), projectInput, pos);
                flattenList.add(false);
                projectPlan.add(column);
                generatePlans.add(projectPlan);
            }
            forEach = new LOForEach(mPlan, new OperatorKey(scope, NodeIdGenerator.getGenerator().getNextNodeId(scope)), generatePlans, flattenList);
            LogicalOperator pred = mPlan.getSuccessors(load).get(0);
            /*mPlan.disconnect(load, pred);
            mPlan.add(forEach);
            mPlan.connect(load, forEach);
            mPlan.connect(forEach, pred);
            forEach.getSchema();*/
            MultiMap<Integer, Column> mappedFields = new MultiMap<Integer, Column>();
            List<Column> columns;
            for (int i=0;i<=load.getSchema().size();i++) {
                columns = new ArrayList<Column>();
                columns.add(new Column(new Pair<Integer, Integer>(0, i)));
                mappedFields.put(i, columns);
            }
            mPlan.add(forEach);
            mPlan.doInsertBetween(load, forEach, pred, false);
            forEach.getProjectionMap().setMappedFields(mappedFields);
            pred.rewire(load, 0, forEach, false);
        }
       
        // Begin to prune
        for (Pair<Integer, Integer> pair: loaderRequiredFields.getFields())
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.logicalLayer.LOForEach

  • org.apache.pig.impl.logicalLayer.optimizer.OpLimitOptimizer
  • org.apache.pig.impl.logicalLayer.optimizer.PruneColumns
  • org.apache.pig.impl.logicalLayer.optimizer.PushDownForeachFlatten
  • org.apache.pig.impl.logicalLayer.optimizer.PushUpFilter
  • org.apache.pig.impl.logicalLayer.optimizer.SchemaRemover
  • org.apache.pig.impl.logicalLayer.optimizer.TypeCastInserter
  • org.apache.pig.impl.logicalLayer.schema.Schema
  • org.apache.pig.impl.plan.ProjectionMap
  • org.apache.pig.impl.plan.RequiredFields
  • org.apache.pig.test.TestRelevantFields
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.