Examples of BucketMapJoinContext

Examples of org.apache.hadoop.hive.ql.plan.BucketMapJoinContext

459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
  }

  private void setUpFetchOpContext(FetchOperator fetchOp, String alias, String currentInputFile)
      throws Exception {

    BucketMapJoinContext bucketMatcherCxt = this.work.getBucketMapjoinContext();

    Class<? extends BucketMatcher> bucketMatcherCls = bucketMatcherCxt.getBucketMatcherClass();
    BucketMatcher bucketMatcher = (BucketMatcher) ReflectionUtils.newInstance(bucketMatcherCls,
        null);
    bucketMatcher.setAliasBucketFileNameMapping(bucketMatcherCxt.getAliasBucketFileNameMapping());

    List<Path> aliasFiles = bucketMatcher.getAliasBucketFiles(currentInputFile, bucketMatcherCxt
        .getMapJoinBigTableAlias(), alias);
    fetchOp.setupContext(aliasFiles);
  }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.BucketMapJoinContext

519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
  private void setUpFetchContexts(String alias, MergeQueue mergeQueue) throws HiveException {
    mergeQueue.clearFetchContext();

    String currentInputFile = getExecContext().getCurrentInputFile();

    BucketMapJoinContext bucketMatcherCxt = localWork.getBucketMapjoinContext();
    Class<? extends BucketMatcher> bucketMatcherCls = bucketMatcherCxt.getBucketMatcherClass();
    BucketMatcher bucketMatcher = ReflectionUtils.newInstance(bucketMatcherCls, null);

    getExecContext().setFileId(bucketMatcherCxt.createFileId(currentInputFile));
    LOG.info("set task id: " + getExecContext().getFileId());

    bucketMatcher.setAliasBucketFileNameMapping(bucketMatcherCxt
        .getAliasBucketFileNameMapping());

    List<Path> aliasFiles = bucketMatcher.getAliasBucketFiles(currentInputFile,
        bucketMatcherCxt.getMapJoinBigTableAlias(), alias);

    mergeQueue.setupContext(aliasFiles);
  }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.BucketMapJoinContext

493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
  private void setUpFetchContexts(String alias, MergeQueue mergeQueue) throws HiveException {
    mergeQueue.clearFetchContext();

    String currentInputFile = getExecContext().getCurrentInputFile();

    BucketMapJoinContext bucketMatcherCxt = localWork.getBucketMapjoinContext();
    Class<? extends BucketMatcher> bucketMatcherCls = bucketMatcherCxt.getBucketMatcherClass();
    BucketMatcher bucketMatcher = ReflectionUtils.newInstance(bucketMatcherCls, null);

    getExecContext().setFileId(bucketMatcherCxt.createFileId(currentInputFile));
    LOG.info("set task id: " + getExecContext().getFileId());

    bucketMatcher.setAliasBucketFileNameMapping(bucketMatcherCxt
        .getAliasBucketFileNameMapping());

    List<Path> aliasFiles = bucketMatcher.getAliasBucketFiles(currentInputFile,
        bucketMatcherCxt.getMapJoinBigTableAlias(), alias);

    mergeQueue.setupContext(aliasFiles);
  }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.BucketMapJoinContext

422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
  }

  private void setUpFetchOpContext(FetchOperator fetchOp, String alias, String currentInputFile)
      throws Exception {

    BucketMapJoinContext bucketMatcherCxt = this.work.getBucketMapjoinContext();

    Class<? extends BucketMatcher> bucketMatcherCls = bucketMatcherCxt.getBucketMatcherClass();
    BucketMatcher bucketMatcher = (BucketMatcher) ReflectionUtils.newInstance(bucketMatcherCls,
        null);
    bucketMatcher.setAliasBucketFileNameMapping(bucketMatcherCxt.getAliasBucketFileNameMapping());

    List<Path> aliasFiles = bucketMatcher.getAliasBucketFiles(currentInputFile, bucketMatcherCxt
        .getMapJoinBigTableAlias(), alias);
    fetchOp.setupContext(aliasFiles);
  }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.BucketMapJoinContext

277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
          plan.setMapLocalWork(null);
          ((SMBMapJoinOperator)currMapJoinOp).getConf().setLocalWork(localPlan);
        } else {
          plan.setMapLocalWork(localPlan);
        }
        BucketMapJoinContext bucketMJCxt = new BucketMapJoinContext();
        localPlan.setBucketMapjoinContext(bucketMJCxt);
        bucketMJCxt.setAliasBucketFileNameMapping(aliasBucketFileNameMapping);
        bucketMJCxt.setBucketFileNameMapping(currMapJoinOp.getConf().getBigTableBucketNumMapping());
        localPlan.setInputFileChangeSensitive(true);
        bucketMJCxt.setMapJoinBigTableAlias(currMapJoinOp.getConf().getBigTableAlias());
        bucketMJCxt.setBucketMatcherClass(org.apache.hadoop.hive.ql.exec.DefaultBucketMatcher.class);
        bucketMJCxt.setBigTablePartSpecToFileMapping(
          currMapJoinOp.getConf().getBigTablePartSpecToFileMapping());
        plan.setUseBucketizedHiveInputFormat(currMapJoinOp instanceof SMBMapJoinOperator);
      }
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.BucketMapJoinContext

520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
  private void setUpFetchContexts(String alias, MergeQueue mergeQueue) throws HiveException {
    mergeQueue.clearFetchContext();

    Path currentInputPath = getExecContext().getCurrentInputPath();

    BucketMapJoinContext bucketMatcherCxt = localWork.getBucketMapjoinContext();
    Class<? extends BucketMatcher> bucketMatcherCls = bucketMatcherCxt.getBucketMatcherClass();
    BucketMatcher bucketMatcher = ReflectionUtils.newInstance(bucketMatcherCls, null);

    getExecContext().setFileId(bucketMatcherCxt.createFileId(currentInputPath.toString()));
    LOG.info("set task id: " + getExecContext().getFileId());

    bucketMatcher.setAliasBucketFileNameMapping(bucketMatcherCxt
        .getAliasBucketFileNameMapping());

    List<Path> aliasFiles = bucketMatcher.getAliasBucketFiles(currentInputPath.toString(),
        bucketMatcherCxt.getMapJoinBigTableAlias(), alias);

    mergeQueue.setupContext(aliasFiles);
  }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.BucketMapJoinContext

454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
  }

  private void setUpFetchOpContext(FetchOperator fetchOp, String alias, String currentInputFile)
      throws Exception {

    BucketMapJoinContext bucketMatcherCxt = this.work.getBucketMapjoinContext();

    Class<? extends BucketMatcher> bucketMatcherCls = bucketMatcherCxt.getBucketMatcherClass();
    BucketMatcher bucketMatcher = ReflectionUtils.newInstance(bucketMatcherCls,
        null);
    bucketMatcher.setAliasBucketFileNameMapping(bucketMatcherCxt.getAliasBucketFileNameMapping());

    List<Path> aliasFiles = bucketMatcher.getAliasBucketFiles(currentInputFile, bucketMatcherCxt
        .getMapJoinBigTableAlias(), alias);
    fetchOp.setupContext(aliasFiles);
  }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.BucketMapJoinContext

109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
            plan.setMapLocalWork(null);
            ((SMBMapJoinOperator) currMapJoinOp).getConf().setLocalWork(localPlan);
          } else {
            plan.setMapLocalWork(localPlan);
          }
          BucketMapJoinContext bucketMJCxt = new BucketMapJoinContext();
          localPlan.setBucketMapjoinContext(bucketMJCxt);
          bucketMJCxt.setAliasBucketFileNameMapping(aliasBucketFileNameMapping);
          bucketMJCxt.setBucketFileNameMapping(
              currMapJoinOp.getConf().getBigTableBucketNumMapping());
          localPlan.setInputFileChangeSensitive(true);
          bucketMJCxt.setMapJoinBigTableAlias(currMapJoinOp.getConf().getBigTableAlias());
          bucketMJCxt
              .setBucketMatcherClass(org.apache.hadoop.hive.ql.exec.DefaultBucketMatcher.class);
          bucketMJCxt.setBigTablePartSpecToFileMapping(
              currMapJoinOp.getConf().getBigTablePartSpecToFileMapping());
          // BucketizedHiveInputFormat should be used for either sort merge join or bucket map join
          if ((currMapJoinOp instanceof SMBMapJoinOperator)
              || (currMapJoinOp.getConf().isBucketMapJoin())) {
            plan.setUseBucketizedHiveInputFormat(true);
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.BucketMapJoinContext

109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
            plan.setMapRedLocalWork(null);
            ((SMBMapJoinOperator) currMapJoinOp).getConf().setLocalWork(localPlan);
          } else {
            plan.setMapRedLocalWork(localPlan);
          }
          BucketMapJoinContext bucketMJCxt = new BucketMapJoinContext();
          localPlan.setBucketMapjoinContext(bucketMJCxt);
          bucketMJCxt.setAliasBucketFileNameMapping(aliasBucketFileNameMapping);
          bucketMJCxt.setBucketFileNameMapping(
              currMapJoinOp.getConf().getBigTableBucketNumMapping());
          localPlan.setInputFileChangeSensitive(true);
          bucketMJCxt.setMapJoinBigTableAlias(currMapJoinOp.getConf().getBigTableAlias());
          bucketMJCxt
              .setBucketMatcherClass(org.apache.hadoop.hive.ql.exec.DefaultBucketMatcher.class);
          bucketMJCxt.setBigTablePartSpecToFileMapping(
              currMapJoinOp.getConf().getBigTablePartSpecToFileMapping());
          // BucketizedHiveInputFormat should be used for either sort merge join or bucket map join
          if ((currMapJoinOp instanceof SMBMapJoinOperator)
              || (currMapJoinOp.getConf().isBucketMapJoin())) {
            plan.setUseBucketizedHiveInputFormat(true);
View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.BucketMapJoinContext

458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
  }

  private void setUpFetchOpContext(FetchOperator fetchOp, String alias, String currentInputFile)
      throws Exception {

    BucketMapJoinContext bucketMatcherCxt = this.work.getBucketMapjoinContext();

    Class<? extends BucketMatcher> bucketMatcherCls = bucketMatcherCxt.getBucketMatcherClass();
    BucketMatcher bucketMatcher = ReflectionUtils.newInstance(bucketMatcherCls,
        null);
    bucketMatcher.setAliasBucketFileNameMapping(bucketMatcherCxt.getAliasBucketFileNameMapping());

    List<Path> aliasFiles = bucketMatcher.getAliasBucketFiles(currentInputFile, bucketMatcherCxt
        .getMapJoinBigTableAlias(), alias);
    fetchOp.setupContext(aliasFiles);
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.