Package org.apache.crunch

Examples of org.apache.crunch.GroupingOptions

400
401
402
403
404
405
406
407
408
409
410
      }
    }, tableOf(HBaseTypes.keyValues(), nulls()));
    List<KeyValue> splitPoints = getSplitPoints(table);
    Path partitionFile = new Path(((DistributedPipeline) kvs.getPipeline()).createTempPath(), "partition");
    writePartitionInfo(conf, partitionFile, splitPoints);
    GroupingOptions options = GroupingOptions.builder()
        .partitionerClass(TotalOrderPartitioner.class)
        .sortComparatorClass(KeyValueComparator.class)
        .conf(TotalOrderPartitioner.PARTITIONER_PATH, partitionFile.toString())
        .numReducers(splitPoints.size() + 1)
        .build();
View Full Code Here
401
402
403
404
405
406
407
408
409
410
411
      }
    }, tableOf(HBaseTypes.keyValues(), nulls()));
    List<KeyValue> splitPoints = getSplitPoints(table);
    Path partitionFile = new Path(((DistributedPipeline) kvs.getPipeline()).createTempPath(), "partition");
    writePartitionInfo(conf, partitionFile, splitPoints);
    GroupingOptions options = GroupingOptions.builder()
        .partitionerClass(TotalOrderPartitioner.class)
        .sortComparatorClass(KeyValueComparator.class)
        .conf(TotalOrderPartitioner.PARTITIONER_PATH, partitionFile.toString())
        .numReducers(splitPoints.size() + 1)
        .build();
View Full Code Here
125
126
127
128
129
130
131
132
      @Override
      public void process(T input, Emitter<Pair<T, Void>> emitter) {
        emitter.emit(Pair.of(input, (Void) null));
      }
    }, type);
    GroupingOptions options = buildGroupingOptions(pt, conf, numReducers, order);
    return pt.groupByKey(options).ungroup().keys();
  }
View Full Code Here
155
156
157
158
159
160
161
162
   *
   * @return a {@code PTable} representing the sorted collection.
   */
  public static <K, V> PTable<K, V> sort(PTable<K, V> table, int numReducers, Order key) {
    Configuration conf = table.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(table, conf, numReducers, key);
    return table.groupByKey(options).ungroup();
  }
View Full Code Here
215
216
217
218
219
220
221
222
      ColumnOrder... columnOrders) {
    PType<T> pType = collection.getPType();
    KeyExtraction<T> ke = new KeyExtraction<T>(pType, columnOrders);
    PTable<Object, T> pt = collection.by(ke.getByFn(), ke.getKeyType());
    Configuration conf = collection.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(pt, conf, numReducers, columnOrders);
    return pt.groupByKey(options).ungroup().values();
  }
View Full Code Here
389
390
391
392
393
394
395
396
397
398
399
      }
    }, tableOf(writables(KeyValue.class), nulls()));
    List <KeyValue> splitPoints = getSplitPoints(table);
    Path partitionFile = new Path(((DistributedPipeline) kvs.getPipeline()).createTempPath(), "partition");
    writePartitionInfo(conf, partitionFile, splitPoints);
    GroupingOptions options = GroupingOptions.builder()
        .partitionerClass(TotalOrderPartitioner.class)
        .conf(TotalOrderPartitioner.PARTITIONER_PATH, partitionFile.toString())
        .numReducers(splitPoints.size() + 1)
        .sortComparatorClass(KeyValueComparator.class)
        .build();
View Full Code Here
124
125
126
127
128
129
130
131
      @Override
      public void process(T input, Emitter<Pair<T, Void>> emitter) {
        emitter.emit(Pair.of(input, (Void) null));
      }
    }, type);
    GroupingOptions options = buildGroupingOptions(pt, conf, numReducers, order);
    return pt.groupByKey(options).ungroup().keys();
  }
View Full Code Here
154
155
156
157
158
159
160
161
   *
   * @return a {@code PTable} representing the sorted collection.
   */
  public static <K, V> PTable<K, V> sort(PTable<K, V> table, int numReducers, Order key) {
    Configuration conf = table.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(table, conf, numReducers, key);
    return table.groupByKey(options).ungroup();
  }
View Full Code Here
214
215
216
217
218
219
220
221
      ColumnOrder... columnOrders) {
    PType<T> pType = collection.getPType();
    SortFns.KeyExtraction<T> ke = new SortFns.KeyExtraction<T>(pType, columnOrders);
    PTable<Object, T> pt = collection.by(ke.getByFn(), ke.getKeyType());
    Configuration conf = collection.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(pt, conf, numReducers, columnOrders);
    return pt.groupByKey(options).ungroup().values();
  }
View Full Code Here
123
124
125
126
127
128
129
130
      @Override
      public void process(T input, Emitter<Pair<T, Void>> emitter) {
        emitter.emit(Pair.of(input, (Void) null));
      }
    }, type);
    GroupingOptions options = buildGroupingOptions(pt, conf, numReducers, order);
    return pt.groupByKey(options).ungroup().keys();
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.GroupingOptions

  • org.apache.crunch.io.hbase.HFileUtils
  • org.apache.crunch.lib.Sort
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.