Examples of SparkComparator

Examples of org.apache.crunch.impl.spark.SparkComparator

117
118
119
120
121
122
123
124
125
126
127
          .mapToPair(new MapOutputFunction(keySerde, valueSerde))
          .groupByKey(numPartitions);
    }

    if (groupingOptions.requireSortedKeys() || groupingOptions.getSortComparatorClass() != null) {
      SparkComparator scmp = new SparkComparator(groupingOptions, ptype, runtime.getRuntimeContext());
      groupedRDD = groupedRDD.sortByKey(scmp);
    }
    if (groupingOptions.getGroupingComparatorClass() != null) {
      groupedRDD = groupedRDD.mapPartitionsToPair(
          new ReduceGroupingFunction(groupingOptions, ptype, runtime.getRuntimeContext()));
View Full Code Here

Examples of org.apache.crunch.impl.spark.SparkComparator

105
106
107
108
109
110
111
112
113
114
115
          .map(new MapOutputFunction(keySerde, valueSerde))
          .groupByKey(numPartitions);
    }

    if (groupingOptions.requireSortedKeys() || groupingOptions.getSortComparatorClass() != null) {
      SparkComparator scmp = new SparkComparator(groupingOptions, ptype, runtime.getRuntimeContext());
      groupedRDD = groupedRDD.sortByKey(scmp);
    }
    if (groupingOptions.getGroupingComparatorClass() != null) {
      groupedRDD = groupedRDD.mapPartitions(
          new ReduceGroupingFunction(groupingOptions, ptype, runtime.getRuntimeContext()));
View Full Code Here

Examples of org.apache.crunch.impl.spark.SparkComparator

115
116
117
118
119
120
121
122
123
124
125
          .map(new MapOutputFunction(keySerde, valueSerde))
          .groupByKey(numPartitions);
    }

    if (groupingOptions.requireSortedKeys() || groupingOptions.getSortComparatorClass() != null) {
      SparkComparator scmp = new SparkComparator(groupingOptions, ptype, runtime.getRuntimeContext());
      groupedRDD = groupedRDD.sortByKey(scmp);
    }
    if (groupingOptions.getGroupingComparatorClass() != null) {
      groupedRDD = groupedRDD.mapPartitions(
          new ReduceGroupingFunction(groupingOptions, ptype, runtime.getRuntimeContext()));
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.