Package org.apache.crunch.impl.spark

Examples of org.apache.crunch.impl.spark.SparkPipeline.readTextFile()

135
136
137
138
139
140
141
142
143
144
145
  @Test
  public void testAvroReflectSortPair() throws IOException {
    Pipeline pipeline = new SparkPipeline("local", "sort");
    pipeline.enableDebug();
    String rsrc = tmpDir.copyResourceFileName("set2.txt");
    PCollection<Pair<String, StringWrapper>> in = pipeline.readTextFile(rsrc)
        .parallelDo(new MapFn<String, Pair<String, StringWrapper>>() {

          @Override
          public Pair<String, StringWrapper> map(String input) {
            return Pair.of(input, wrap(input));
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
  }

  @Test
  public void testAvroReflectSortTable() throws IOException {
    Pipeline pipeline = new SparkPipeline("local", "sort");
    PTable<String, StringWrapper> unsorted = pipeline.readTextFile(tmpDir.copyResourceFileName("set2.txt")).parallelDo(
        new MapFn<String, Pair<String, StringWrapper>>() {

          @Override
          public Pair<String, StringWrapper> map(String input) {
            return Pair.of(input, wrap(input));
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.