Examples of mapPartitions()

Examples of org.apache.spark.api.java.JavaPairRDD.mapPartitions()

56
57
58
59
60
61
62
63
64
65
66
    List<PCollectionImpl<?>> parents = getParents();
    JavaRDD[] rdds = new JavaRDD[parents.size()];
    for (int i = 0; i < rdds.length; i++) {
      if (parents.get(i) instanceof PTableBase) {
        JavaPairRDD prdd = (JavaPairRDD) ((SparkCollection) parents.get(i)).getJavaRDDLike(runtime);
        rdds[i] = prdd.mapPartitions(new FlatMapPairDoFn(IdentityFn.getInstance(), runtime.getRuntimeContext()));
      } else {
        rdds[i] = (JavaRDD) ((SparkCollection) parents.get(i)).getJavaRDDLike(runtime);
      }
    }
    return runtime.getSparkContext().union(rdds);
View Full Code Here

Examples of org.apache.spark.api.java.JavaRDD.mapPartitions()

57
58
59
60
61
62
63
64
65
66
67
    for (int i = 0; i < rdds.length; i++) {
      if (parents.get(i) instanceof PTableBase) {
        rdds[i] = (JavaPairRDD) ((SparkCollection) parents.get(i)).getJavaRDDLike(runtime);
      } else {
        JavaRDD rdd = (JavaRDD) ((SparkCollection) parents.get(i)).getJavaRDDLike(runtime);
        rdds[i] = rdd.mapPartitions(new PairFlatMapDoFn(IdentityFn.getInstance(), runtime.getRuntimeContext()));
      }
    }
    return runtime.getSparkContext().union(rdds);
  }
}
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.