Examples of LOSort

Examples of org.apache.pig.impl.logicalLayer.LOSort

241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
            {
                if(mode == ExecType.LOCAL) {
                    //We don't need this optimisation to happen in the local mode.
                    //so we do nothing here.
                } else {
                    LOSort sort = (LOSort)predecessor;
                    if (sort.getLimit()==-1)
                        sort.setLimit(limit.getLimit());
                    else
                        sort.setLimit(sort.getLimit()<limit.getLimit()?sort.getLimit():limit.getLimit());
                    try {
                        mPlan.removeAndReconnect(limit);
                    } catch (Exception e) {
                        int errCode = 2012;
                        String msg = "Can not remove LOLimit after LOSort";
                        throw new OptimizerException(msg, errCode, PigException.BUG, e);
                    }
                }
            }
            // Limit is merged into another LOLimit
            else if (predecessor instanceof LOLimit)
            {
              LOLimit beforeLimit = (LOLimit)predecessor;
              beforeLimit.setLimit(beforeLimit.getLimit()<limit.getLimit()?beforeLimit.getLimit():limit.getLimit());
              try {
                mPlan.removeAndReconnect(limit);
              } catch (Exception e) {
                  int errCode = 2012;
                  String msg = "Can not remove LOLimit after LOLimit";
                throw new OptimizerException(msg, errCode, PigException.BUG, e);
              }
            }
            // Limit and OrderBy (LOSort) can be separated by split
            else if (predecessor instanceof LOSplitOutput) {              
                if(mode == ExecType.LOCAL) {
                    //We don't need this optimisation to happen in the local mode.
                    //so we do nothing here.
                } else {
                    List<LogicalOperator> grandparants = mPlan
                            .getPredecessors(predecessor);
                    // After insertion of splitters, any node in the plan can
                    // have at most one predecessor
                    if (grandparants != null && grandparants.size() != 0
                            && grandparants.get(0) instanceof LOSplit) {                       
                        List<LogicalOperator> greatGrandparants = mPlan
                                .getPredecessors(grandparants.get(0));
                        if (greatGrandparants != null
                                && greatGrandparants.size() != 0
                                && greatGrandparants.get(0) instanceof LOSort) {                          
                            LOSort sort = (LOSort)greatGrandparants.get(0);
                            LOSort newSort = new LOSort(
                                    sort.getPlan(),
                                    new OperatorKey(
                                            sort.getOperatorKey().scope,
                                            NodeIdGenerator
                                                    .getGenerator()
                                                    .getNextNodeId(
                                                            sort.getOperatorKey().scope)),
                                    sort.getSortColPlans(),
                                    sort.getAscendingCols(),
                                    sort.getUserFunc());
                                                 
                            newSort.setLimit(limit.getLimit());
                            try {
                                mPlan.replace(limit, newSort);
                            } catch (PlanException e) {
                                int errCode = 2012;
                                String msg = "Can not replace LOLimit with LOSort after splitter";
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSort

220
221
222
223
224
225
226
227
228
229
230
231
            // distinct(flatten({(1), (1)})) is (1). However,
            // flatten(distinct({(1), (1)})) is (1), (1)
           
            // in both cases correctness is not affected
            if(successor instanceof LOSort) {
                LOSort sort = (LOSort) successor;
                RequiredFields sortRequiredField = sort.getRequiredFields().get(0);
               
                if(sortRequiredField.getNeedAllFields()) {
                    return false;
                }
               
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSort

277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
    @Test
    public void testQueryOrderByNoSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a';");
        LogicalPlan lp = planTester.buildPlan("b = order a by $1;");
       
        LOSort sort = (LOSort)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields sortRelevantFields0 = sort.getRelevantInputs(0, 0).get(0);
        assertTrue(sortRelevantFields0.getNeedAllFields() == false);
        assertTrue(sortRelevantFields0.getNeedNoFields() == false);
        assertTrue(sortRelevantFields0.getFields().size() == 1);
        assertTrue(sortRelevantFields0.getFields().contains(new Pair<Integer, Integer>(0, 0)));
       
        RequiredFields sortRelevantFields1 = sort.getRelevantInputs(0, 1).get(0);
        assertTrue(sortRelevantFields1.getNeedAllFields() == false);
        assertTrue(sortRelevantFields1.getNeedNoFields() == false);
        assertTrue(sortRelevantFields1.getFields().size() == 1);
        assertTrue(sortRelevantFields1.getFields().contains(new Pair<Integer, Integer>(0, 1)));
    }
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSort

531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
    @Test
    public void testQueryOrderByWithSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a' as (url,hitCount);");
        LogicalPlan lp = planTester.buildPlan("b = order a by $1;");
       
        LOSort sort = (LOSort)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields sortRelevantFields0 = sort.getRelevantInputs(0, 0).get(0);
        assertTrue(sortRelevantFields0.getNeedAllFields() == false);
        assertTrue(sortRelevantFields0.getNeedNoFields() == false);
        assertTrue(sortRelevantFields0.getFields().size() == 1);
        assertTrue(sortRelevantFields0.getFields().contains(new Pair<Integer, Integer>(0, 0)));
       
        RequiredFields sortRelevantFields1 = sort.getRelevantInputs(0, 1).get(0);
        assertTrue(sortRelevantFields1.getNeedAllFields() == false);
        assertTrue(sortRelevantFields1.getNeedNoFields() == false);
        assertTrue(sortRelevantFields1.getFields().size() == 1);
        assertTrue(sortRelevantFields1.getFields().contains(new Pair<Integer, Integer>(0, 1)));
    }
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSort

927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
    @Test
    public void testQueryOrderByStarNoSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a';");
        LogicalPlan lp = planTester.buildPlan("b = order a by *;");
       
        LOSort sort = (LOSort)lp.getLeaves().get(0);
        RequiredFields sortRelevantFields0 = sort.getRelevantInputs(0, 0).get(0);
        assertTrue(sortRelevantFields0.getNeedAllFields() == false);
        assertTrue(sortRelevantFields0.getNeedNoFields() == false);
        assertTrue(sortRelevantFields0.getFields().size() == 1);
        assertTrue(sortRelevantFields0.getFields().get(0).first == 0);
        assertTrue(sortRelevantFields0.getFields().get(0).second == 0);
       
        RequiredFields sortRelevantFields1 = sort.getRelevantInputs(0, 1).get(0);
        assertTrue(sortRelevantFields1.getNeedAllFields() == false);
        assertTrue(sortRelevantFields1.getNeedNoFields() == false);
        assertTrue(sortRelevantFields1.getFields().size() == 1);
        assertTrue(sortRelevantFields1.getFields().get(0).first == 0);
        assertTrue(sortRelevantFields1.getFields().get(0).second == 1);
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSort

1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
    @Test
    public void testQueryOrderByStarWithSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a' as (url,hitCount);");
        LogicalPlan lp = planTester.buildPlan("b = order a by *;");
       
        LOSort sort = (LOSort)lp.getLeaves().get(0);
        RequiredFields sortRelevantFields0 = sort.getRelevantInputs(0, 0).get(0);
        assertTrue(sortRelevantFields0.getNeedAllFields() == false);
        assertTrue(sortRelevantFields0.getNeedNoFields() == false);
        assertTrue(sortRelevantFields0.getFields().size() == 1);
        assertTrue(sortRelevantFields0.getFields().contains(new Pair<Integer, Integer>(0, 0)));
       
        RequiredFields sortRelevantFields1 = sort.getRelevantInputs(0, 1).get(0);
        assertTrue(sortRelevantFields1.getNeedAllFields() == false);
        assertTrue(sortRelevantFields1.getNeedNoFields() == false);
        assertTrue(sortRelevantFields1.getFields().size() == 1);
        assertTrue(sortRelevantFields1.getFields().contains(new Pair<Integer, Integer>(0, 1)));
    }
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSort

237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
            {
                if(mode == ExecType.LOCAL) {
                    //We don't need this optimisation to happen in the local mode.
                    //so we do nothing here.
                } else {
                    LOSort sort = (LOSort)predecessor;
                    if (sort.getLimit()==-1)
                        sort.setLimit(limit.getLimit());
                    else
                        sort.setLimit(sort.getLimit()<limit.getLimit()?sort.getLimit():limit.getLimit());
                    try {
                        mPlan.removeAndReconnect(limit);
                    } catch (Exception e) {
                        int errCode = 2012;
                        String msg = "Can not remove LOLimit after LOSort";
                        throw new OptimizerException(msg, errCode, PigException.BUG, e);
                    }
                }
            }
            // Limit is merged into another LOLimit
            else if (predecessor instanceof LOLimit)
            {
              LOLimit beforeLimit = (LOLimit)predecessor;
              beforeLimit.setLimit(beforeLimit.getLimit()<limit.getLimit()?beforeLimit.getLimit():limit.getLimit());
              try {
                mPlan.removeAndReconnect(limit);
              } catch (Exception e) {
                  int errCode = 2012;
                  String msg = "Can not remove LOLimit after LOLimit";
                throw new OptimizerException(msg, errCode, PigException.BUG, e);
              }
            }
            // Limit and OrderBy (LOSort) can be separated by split
            else if (predecessor instanceof LOSplitOutput) {              
                if(mode == ExecType.LOCAL) {
                    //We don't need this optimisation to happen in the local mode.
                    //so we do nothing here.
                } else {
                    List<LogicalOperator> grandparants = mPlan
                            .getPredecessors(predecessor);
                    // After insertion of splitters, any node in the plan can
                    // have at most one predecessor
                    if (grandparants != null && grandparants.size() != 0
                            && grandparants.get(0) instanceof LOSplit) {                       
                        List<LogicalOperator> greatGrandparants = mPlan
                                .getPredecessors(grandparants.get(0));
                        if (greatGrandparants != null
                                && greatGrandparants.size() != 0
                                && greatGrandparants.get(0) instanceof LOSort) {                          
                            LOSort sort = (LOSort)greatGrandparants.get(0);
                            LOSort newSort = new LOSort(
                                    sort.getPlan(),
                                    new OperatorKey(
                                            sort.getOperatorKey().scope,
                                            NodeIdGenerator
                                                    .getGenerator()
                                                    .getNextNodeId(
                                                            sort.getOperatorKey().scope)),
                                    sort.getSortColPlans(),
                                    sort.getAscendingCols(),
                                    sort.getUserFunc());
                                                 
                            newSort.setLimit(limit.getLimit());
                            try {
                                mPlan.replace(limit, newSort);
                            } catch (PlanException e) {
                                int errCode = 2012;
                                String msg = "Can not replace LOLimit with LOSort after splitter";
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSort

205
206
207
208
209
210
211
212
213
214
215
                                              lo.getRequestedParallelism());
            connectInputs(compiledInputs, pom, execEngine.getPhysicalOpTable());
            return pom.getOperatorKey();
        }
        else if (lo instanceof LOSort) {
            LOSort loSort = (LOSort) lo;
            //must break up into 2 map reduce jobs, one for gathering quantiles, another for sorting
            POMapreduce quantileJob = getQuantileJob(lo.getScope(),
                                                     nodeIdGenerator.getNextNodeId(logicalKey.getScope()),
                                                     execEngine.getPhysicalOpTable(),
                                                     logicalKey,
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSort

196
197
198
199
200
201
202
203
204
205
206
207
            // distinct(flatten({(1), (1)})) is (1). However,
            // flatten(distinct({(1), (1)})) is (1), (1)
           
            // in both cases correctness is not affected
            if(successor instanceof LOSort) {
                LOSort sort = (LOSort) successor;
                RequiredFields sortRequiredField = sort.getRequiredFields().get(0);
               
                if(sortRequiredField.getNeedAllFields()) {
                    return false;
                }
               
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOSort

205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
            {
                if(mode == ExecType.LOCAL) {
                    //We don't need this optimisation to happen in the local mode.
                    //so we do nothing here.
                } else {
                    LOSort sort = (LOSort)predecessor;
                    if (sort.getLimit()==-1)
                        sort.setLimit(limit.getLimit());
                    else
                        sort.setLimit(sort.getLimit()<limit.getLimit()?sort.getLimit():limit.getLimit());
                    try {
                        mPlan.removeAndReconnect(limit);
                    } catch (Exception e) {
                        int errCode = 2012;
                        String msg = "Can not remove LOLimit after LOSort";
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.