Examples of nulls()

Examples of org.apache.crunch.types.PTypeFamily.nulls()

310
311
312
313
314
315
316
317
318
319
320
    PTypeFamily tf = collect.getTypeFamily();
    return collect.parallelDo("Aggregate.aggregator", new MapFn<S, Pair<Void, S>>() {
      public Pair<Void, S> map(S input) {
        return Pair.of(null, input);
      }
    }, tf.tableOf(tf.nulls(), collect.getPType()))
    .groupByKey(1)
    .combineValues(aggregator)
    .values();
  }
}
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.nulls()

117
118
119
120
121
122
123
124
125
126
127
   *
   * @return a {@code PCollection} representing the sorted collection.
   */
  public static <T> PCollection<T> sort(PCollection<T> collection, int numReducers, Order order) {
    PTypeFamily tf = collection.getTypeFamily();
    PTableType<T, Void> type = tf.tableOf(collection.getPType(), tf.nulls());
    Configuration conf = collection.getPipeline().getConfiguration();
    PTable<T, Void> pt = collection.parallelDo("sort-pre", new DoFn<T, Pair<T, Void>>() {
      @Override
      public void process(T input, Emitter<Pair<T, Void>> emitter) {
        emitter.emit(Pair.of(input, (Void) null));
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.nulls()

67
68
69
70
71
72
73
74
75
76
77
  public static <S> PCollection<S> distinct(PCollection<S> input, int flushEvery) {
    Preconditions.checkArgument(flushEvery > 0);
    PType<S> pt = input.getPType();
    PTypeFamily ptf = pt.getFamily();
    return input
        .parallelDo("pre-distinct", new PreDistinctFn<S>(flushEvery, pt), ptf.tableOf(pt, ptf.nulls()))
        .groupByKey()
        .parallelDo("post-distinct", new PostDistinctFn<S>(), pt);
  }
 
  /**
 
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.nulls()

67
68
69
70
71
72
73
74
75
76
77
  public static <S> PCollection<S> distinct(PCollection<S> input, int flushEvery) {
    Preconditions.checkArgument(flushEvery > 0);
    PType<S> pt = input.getPType();
    PTypeFamily ptf = pt.getFamily();
    return input
        .parallelDo("pre-distinct", new PreDistinctFn<S>(flushEvery, pt), ptf.tableOf(pt, ptf.nulls()))
        .groupByKey()
        .parallelDo("post-distinct", new PostDistinctFn<S>(), pt);
  }
 
  /**
 
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.nulls()

306
307
308
309
310
311
312
313
314
315
316
    PTypeFamily tf = collect.getTypeFamily();
    return collect.parallelDo("Aggregate.aggregator", new MapFn<S, Pair<Void, S>>() {
      public Pair<Void, S> map(S input) {
        return Pair.of(null, input);
      }
    }, tf.tableOf(tf.nulls(), collect.getPType()))
    .groupByKey(1)
    .combineValues(aggregator)
    .values();
  }
}
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.nulls()

116
117
118
119
120
121
122
123
124
125
126
   *
   * @return a {@code PCollection} representing the sorted collection.
   */
  public static <T> PCollection<T> sort(PCollection<T> collection, int numReducers, Order order) {
    PTypeFamily tf = collection.getTypeFamily();
    PTableType<T, Void> type = tf.tableOf(collection.getPType(), tf.nulls());
    Configuration conf = collection.getPipeline().getConfiguration();
    PTable<T, Void> pt = collection.parallelDo("sort-pre", new DoFn<T, Pair<T, Void>>() {
      @Override
      public void process(T input, Emitter<Pair<T, Void>> emitter) {
        emitter.emit(Pair.of(input, (Void) null));
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.nulls()

115
116
117
118
119
120
121
122
123
124
125
   *
   * @return a {@code PCollection} representing the sorted collection.
   */
  public static <T> PCollection<T> sort(PCollection<T> collection, int numReducers, Order order) {
    PTypeFamily tf = collection.getTypeFamily();
    PTableType<T, Void> type = tf.tableOf(collection.getPType(), tf.nulls());
    Configuration conf = collection.getPipeline().getConfiguration();
    PTable<T, Void> pt = collection.parallelDo("sort-pre", new DoFn<T, Pair<T, Void>>() {
      @Override
      public void process(T input, Emitter<Pair<T, Void>> emitter) {
        emitter.emit(Pair.of(input, (Void) null));
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.nulls()

109
110
111
112
113
114
115
116
117
118
119
   *
   * @return a {@link PCollection} representing the sorted collection.
   */
  public static <T> PCollection<T> sort(PCollection<T> collection, Order order) {
    PTypeFamily tf = collection.getTypeFamily();
    PTableType<T, Void> type = tf.tableOf(collection.getPType(), tf.nulls());
    Configuration conf = collection.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(conf, tf, collection.getPType(), order);
    PTable<T, Void> pt = collection.parallelDo("sort-pre", new DoFn<T, Pair<T, Void>>() {
      @Override
      public void process(T input, Emitter<Pair<T, Void>> emitter) {
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.nulls()

163
164
165
166
167
168
169
170
171
172
173
    // sorting
    PTypeFamily tf = collection.getTypeFamily();
    PType<Pair<U, V>> pType = collection.getPType();
    @SuppressWarnings("unchecked")
    PTableType<Pair<U, V>, Void> type = tf.tableOf(tf.pairs(pType.getSubTypes().get(0), pType.getSubTypes().get(1)),
        tf.nulls());
    PTable<Pair<U, V>, Void> pt = collection.parallelDo(new DoFn<Pair<U, V>, Pair<Pair<U, V>, Void>>() {
      @Override
      public void process(Pair<U, V> input, Emitter<Pair<Pair<U, V>, Void>> emitter) {
        emitter.emit(Pair.of(input, (Void) null));
      }
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.nulls()

193
194
195
196
197
198
199
200
201
202
203
      ColumnOrder... columnOrders) {
    PTypeFamily tf = collection.getTypeFamily();
    PType<Tuple3<V1, V2, V3>> pType = collection.getPType();
    @SuppressWarnings("unchecked")
    PTableType<Tuple3<V1, V2, V3>, Void> type = tf.tableOf(
        tf.triples(pType.getSubTypes().get(0), pType.getSubTypes().get(1), pType.getSubTypes().get(2)), tf.nulls());
    PTable<Tuple3<V1, V2, V3>, Void> pt = collection.parallelDo(
        new DoFn<Tuple3<V1, V2, V3>, Pair<Tuple3<V1, V2, V3>, Void>>() {
          @Override
          public void process(Tuple3<V1, V2, V3> input, Emitter<Pair<Tuple3<V1, V2, V3>, Void>> emitter) {
            emitter.emit(Pair.of(input, (Void) null));
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.