Examples of PartitionedMapOutputFunction

Examples of org.apache.crunch.impl.spark.fn.PartitionedMapOutputFunction

106
107
108
109
110
111
112
113
114
115
116
    JavaPairRDD<ByteArray, List<byte[]>> groupedRDD;
    if (groupingOptions.getPartitionerClass() != null) {
      groupedRDD = parentRDD
          .map(new PairMapFunction(ptype.getOutputMapFn(), runtime.getRuntimeContext()))
          .mapToPair(
              new PartitionedMapOutputFunction(keySerde, valueSerde, ptype, groupingOptions.getPartitionerClass(),
              numPartitions, runtime.getRuntimeContext()))
          .groupByKey(new SparkPartitioner(numPartitions));
    } else {
      groupedRDD = parentRDD
          .map(new PairMapFunction(ptype.getOutputMapFn(), runtime.getRuntimeContext()))
View Full Code Here

Examples of org.apache.crunch.impl.spark.fn.PartitionedMapOutputFunction

94
95
96
97
98
99
100
101
102
103
104
    JavaPairRDD<ByteArray, List<byte[]>> groupedRDD;
    if (groupingOptions.getPartitionerClass() != null) {
      groupedRDD = parentRDD
          .map(new PairMapFunction(ptype.getOutputMapFn(), runtime.getRuntimeContext()))
          .map(new PartitionedMapOutputFunction(keySerde, valueSerde, ptype, groupingOptions.getPartitionerClass(),
              numPartitions, runtime.getRuntimeContext()))
          .groupByKey(new SparkPartitioner(numPartitions));
    } else {
      groupedRDD = parentRDD
          .map(new PairMapFunction(ptype.getOutputMapFn(), runtime.getRuntimeContext()))
View Full Code Here

Examples of org.apache.crunch.impl.spark.fn.PartitionedMapOutputFunction

104
105
106
107
108
109
110
111
112
113
114
    JavaPairRDD<ByteArray, List<byte[]>> groupedRDD;
    if (groupingOptions.getPartitionerClass() != null) {
      groupedRDD = parentRDD
          .map(new PairMapFunction(ptype.getOutputMapFn(), runtime.getRuntimeContext()))
          .map(new PartitionedMapOutputFunction(keySerde, valueSerde, ptype, groupingOptions.getPartitionerClass(),
              numPartitions, runtime.getRuntimeContext()))
          .groupByKey(new SparkPartitioner(numPartitions));
    } else {
      groupedRDD = parentRDD
          .map(new PairMapFunction(ptype.getOutputMapFn(), runtime.getRuntimeContext()))
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.