Package org.apache.crunch.types

Examples of org.apache.crunch.types.PTypeFamily.collections()

262
263
264
265
266
267
268
269
            for (V value : values) {
              collected.add(valueType.getDetachedValue(value));
            }
            return collected;
          }
        }, tf.tableOf(collect.getKeyType(), tf.collections(collect.getValueType())));
  }
}
View Full Code Here
265
266
267
268
269
270
271
272
            for (V value : values) {
              collected.add(valueType.getDetachedValue(value));
            }
            return collected;
          }
        }, tf.tableOf(collect.getKeyType(), tf.collections(collect.getValueType())));
  }
}
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
    PTable<K, Pair<U, V>> cgRight = right.parallelDo("coGroupTag2", new CogroupFn2<K, U, V>(),
        ptf.tableOf(keyType, itype));

    PTable<K, Pair<U, V>> both = cgLeft.union(cgRight);

    PType<Pair<Collection<U>, Collection<V>>> otype = ptf.pairs(ptf.collections(leftType), ptf.collections(rightType));
    return both.groupByKey().parallelDo("cogroup", new PostGroupFn<K, U, V>(), ptf.tableOf(keyType, otype));
  }

  private static class CogroupFn1<K, V, U> extends MapValuesFn<K, V, Pair<V, U>> {
    @Override
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
    PTable<K, Pair<U, V>> cgRight = right.parallelDo("coGroupTag2", new CogroupFn2<K, U, V>(),
        ptf.tableOf(keyType, itype));

    PTable<K, Pair<U, V>> both = cgLeft.union(cgRight);

    PType<Pair<Collection<U>, Collection<V>>> otype = ptf.pairs(ptf.collections(leftType), ptf.collections(rightType));
    return both.groupByKey().parallelDo("cogroup", new PostGroupFn<K, U, V>(), ptf.tableOf(keyType, otype));
  }

  private static class CogroupFn1<K, V, U> extends MapValuesFn<K, V, Pair<V, U>> {
    @Override
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
    PType<Pair<V1, V2>> valueType = input.getValueType();
    PTableType<Pair<K, V1>, Pair<V1, V2>> inter = ptf.tableOf(
        ptf.pairs(input.getKeyType(), valueType.getSubTypes().get(0)),
        valueType);
    PTableType<K, Collection<Pair<V1, V2>>> out = ptf.tableOf(input.getKeyType(),
        ptf.collections(input.getValueType()));
    return input.parallelDo("SecondarySort.format", new SSFormatFn<K, V1, V2>(), inter)
        .groupByKey(
            GroupingOptions.builder()
            .groupingComparatorClass(JoinUtils.getGroupingComparator(ptf))
            .partitionerClass(JoinUtils.getPartitionerClass(ptf))
View Full Code Here
298
299
300
301
302
303
304
305
306
307
308
            for (V value : values) {
              collected.add(valueType.getDetachedValue(value));
            }
            return collected;
          }
        }, tf.collections(collect.getValueType()));
  }
 
  public static <S> PCollection<S> aggregate(PCollection<S> collect, Aggregator<S> aggregator) {
    PTypeFamily tf = collect.getTypeFamily();
    return collect.parallelDo("Aggregate.aggregator", new MapFn<S, Pair<Boolean, S>>() {
View Full Code Here
226
227
228
229
230
231
232
233
        for (V value : values) {
          collected.add(valueType.getDetachedValue(value));
        }
        return collected;
      }
    }, tf.tableOf(collect.getKeyType(), tf.collections(collect.getValueType())));
  }
}
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.