Examples of MergeWork

Examples of org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork

453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
   * @throws HiveException
   */
  private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc)
      throws HiveException {
    // merge work only needs input and output.
    MergeWork mergeWork = new MergeWork(mergeFilesDesc.getInputDir(),
        mergeFilesDesc.getOutputDir());
    mergeWork.setListBucketingCtx(mergeFilesDesc.getLbCtx());
    mergeWork.resolveConcatenateMerge(db.getConf());
    mergeWork.setMapperCannotSpanPartns(true);
    DriverContext driverCxt = new DriverContext();
    BlockMergeTask taskExec = new BlockMergeTask();
    taskExec.initialize(db.getConf(), null, driverCxt);
    taskExec.setWork(mergeWork);
    taskExec.setQueryPlan(this.getQueryPlan());
View Full Code Here

Examples of org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork

517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
      if (!hasDynamicPartitions
          && !isSkewedStoredAsDirs(fsInputDesc)) {
        inputDirs.add(inputDir);
      }

      MergeWork work = new MergeWork(inputDirs, finalName,
          hasDynamicPartitions, fsInputDesc.getDynPartCtx());
      LinkedHashMap<String, ArrayList<String>> pathToAliases =
          new LinkedHashMap<String, ArrayList<String>>();
      pathToAliases.put(inputDir, (ArrayList<String>) inputDirs.clone());
      work.setMapperCannotSpanPartns(true);
      work.setPathToAliases(pathToAliases);
      work.setAliasToWork(
          new LinkedHashMap<String, Operator<? extends OperatorDesc>>());
      if (hasDynamicPartitions
          || isSkewedStoredAsDirs(fsInputDesc)) {
        work.getPathToPartitionInfo().put(inputDir,
            new PartitionDesc(tblDesc, null));
      }
      work.setListBucketingCtx(fsInputDesc.getLbCtx());

      return work;
    }

    throw new SemanticException("createRCFileMergeTask called on non-RCFile table");
View Full Code Here

Examples of org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork

478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
      ArrayList<String> inputDirs = new ArrayList<String>();
      if (!hasDynamicPartitions) {
        inputDirs.add(inputDir);
      }

      MergeWork work = new MergeWork(inputDirs, finalName,
          hasDynamicPartitions);
      LinkedHashMap<String, ArrayList<String>> pathToAliases =
          new LinkedHashMap<String, ArrayList<String>>();
      pathToAliases.put(inputDir, (ArrayList<String>) inputDirs.clone());
      work.setMapperCannotSpanPartns(true);
      work.setPathToAliases(pathToAliases);
      work.setAliasToWork(
          new LinkedHashMap<String, Operator<? extends Serializable>>());
      if (hasDynamicPartitions) {
        work.getPathToPartitionInfo().put(inputDir,
            new PartitionDesc(tblDesc, null));
      }

      return work;
    }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork

396
397
398
399
400
401
402
403
404
405
406
   * @throws HiveException
   */
  private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc)
      throws HiveException {
    // merge work only needs input and output.
    MergeWork mergeWork = new MergeWork(mergeFilesDesc.getInputDir(),
        mergeFilesDesc.getOutputDir());
    DriverContext driverCxt = new DriverContext();
    BlockMergeTask taskExec = new BlockMergeTask();
    taskExec.initialize(db.getConf(), null, driverCxt);
    taskExec.setWork(mergeWork);
View Full Code Here

Examples of org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork

496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
   * @throws HiveException
   */
  private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc)
      throws HiveException {
    // merge work only needs input and output.
    MergeWork mergeWork = new MergeWork(mergeFilesDesc.getInputDir(),
        mergeFilesDesc.getOutputDir());
    mergeWork.setListBucketingCtx(mergeFilesDesc.getLbCtx());
    mergeWork.resolveConcatenateMerge(db.getConf());
    mergeWork.setMapperCannotSpanPartns(true);
    DriverContext driverCxt = new DriverContext();
    BlockMergeTask taskExec = new BlockMergeTask();
    taskExec.initialize(db.getConf(), null, driverCxt);
    taskExec.setWork(mergeWork);
    taskExec.setQueryPlan(this.getQueryPlan());
View Full Code Here

Examples of org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork

460
461
462
463
464
465
466
467
468
469
470
   * @throws HiveException
   */
  private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc)
      throws HiveException {
    // merge work only needs input and output.
    MergeWork mergeWork = new MergeWork(mergeFilesDesc.getInputDir(),
        mergeFilesDesc.getOutputDir());
    DriverContext driverCxt = new DriverContext();
    BlockMergeTask taskExec = new BlockMergeTask();
    taskExec.initialize(db.getConf(), null, driverCxt);
    taskExec.setWork(mergeWork);
View Full Code Here

Examples of org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork

617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
      ArrayList<String> inputDirs = new ArrayList<String>();
      if (!hasDynamicPartitions) {
        inputDirs.add(inputDir);
      }

      MergeWork work = new MergeWork(inputDirs, finalName,
          hasDynamicPartitions, fsInputDesc.getDynPartCtx());
      LinkedHashMap<String, ArrayList<String>> pathToAliases =
          new LinkedHashMap<String, ArrayList<String>>();
      pathToAliases.put(inputDir, (ArrayList<String>) inputDirs.clone());
      work.setMapperCannotSpanPartns(true);
      work.setPathToAliases(pathToAliases);
      work.setAliasToWork(
          new LinkedHashMap<String, Operator<? extends OperatorDesc>>());
      if (hasDynamicPartitions) {
        work.getPathToPartitionInfo().put(inputDir,
            new PartitionDesc(tblDesc, null));
      }

      return work;
    }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork

485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
      ArrayList<String> inputDirs = new ArrayList<String>();
      if (!hasDynamicPartitions) {
        inputDirs.add(inputDir);
      }

      MergeWork work = new MergeWork(inputDirs, finalName,
          hasDynamicPartitions);
      LinkedHashMap<String, ArrayList<String>> pathToAliases =
          new LinkedHashMap<String, ArrayList<String>>();
      pathToAliases.put(inputDir, (ArrayList<String>) inputDirs.clone());
      work.setMapperCannotSpanPartns(true);
      work.setPathToAliases(pathToAliases);
      work.setAliasToWork(
          new LinkedHashMap<String, Operator<? extends Serializable>>());
      if (hasDynamicPartitions) {
        work.getPathToPartitionInfo().put(inputDir,
            new PartitionDesc(tblDesc, null));
      }

      return work;
    }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork

434
435
436
437
438
439
440
441
442
443
444
   * @throws HiveException
   */
  private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc)
      throws HiveException {
    // merge work only needs input and output.
    MergeWork mergeWork = new MergeWork(mergeFilesDesc.getInputDir(),
        mergeFilesDesc.getOutputDir());
    DriverContext driverCxt = new DriverContext();
    BlockMergeTask taskExec = new BlockMergeTask();
    taskExec.initialize(db.getConf(), null, driverCxt);
    taskExec.setWork(mergeWork);
View Full Code Here

Examples of org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork

485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
      ArrayList<String> inputDirs = new ArrayList<String>();
      if (!hasDynamicPartitions) {
        inputDirs.add(inputDir);
      }

      MergeWork work = new MergeWork(inputDirs, finalName,
          hasDynamicPartitions);
      LinkedHashMap<String, ArrayList<String>> pathToAliases =
          new LinkedHashMap<String, ArrayList<String>>();
      pathToAliases.put(inputDir, (ArrayList<String>) inputDirs.clone());
      work.setMapperCannotSpanPartns(true);
      work.setPathToAliases(pathToAliases);
      work.setAliasToWork(
          new LinkedHashMap<String, Operator<? extends Serializable>>());
      if (hasDynamicPartitions) {
        work.getPathToPartitionInfo().put(inputDir,
            new PartitionDesc(tblDesc, null));
      }

      return work;
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.