Package org.apache.crunch.impl.spark

Examples of org.apache.crunch.impl.spark.SparkPipeline.done()

73
74
75
76
77
78
79
80
81
82
83
    expectedPairs.add(Pair.of("c", 1L));
    expectedPairs.add(Pair.of("d", 1L));

    assertEquals(expectedPairs, unionValues);

    pipeline.done();
  }

  @Test
  public void testMultiWrite() throws Exception {
    String inputPath = tempDir.copyResourceFileName("set1.txt");
View Full Code Here
110
111
112
113
114
115
116
117
    expectedPairs.add(Pair.of("c", 1L));
    expectedPairs.add(Pair.of("d", 1L));

    assertEquals(expectedPairs, values);

    pipeline.done();
  }
}
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
    wcnt.write(new MySeqFileTableSourceTarget(out, ptt));
    pipeline.run();

    PTable<Text, LongWritable> wcntIn = pipeline.read(new MySeqFileTableSourceTarget(out, ptt));
    assertEquals(new LongWritable(1L), wcntIn.materialize().iterator().next().second());
    pipeline.done();
  }

  static class ToWritables extends MapFn<Pair<String, Long>, Pair<Text, LongWritable>> {
    @Override
    public Pair<Text, LongWritable> map(Pair<String, Long> input) {
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
    PTable<Integer, Pair<String, String>> joined = mapsideJoin.join(customerTable, filteredOrderTable, JoinType.INNER_JOIN);

    List<Pair<Integer, Pair<String, String>>> materializedJoin = Lists.newArrayList(joined.materialize());

    assertTrue(materializedJoin.isEmpty());
    pipeline.done();
  }

  @Test
  public void testMapsideJoin() throws IOException {
    runMapsideJoin(new SparkPipeline("local", "mapside"), false);
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
    expectedPairs.add(Pair.of("c", 1L));
    expectedPairs.add(Pair.of("d", 1L));

    assertEquals(expectedPairs, unionValues);

    pipeline.done();
  }

  @Test
  public void testMultiGroupBy() throws Exception {
    String inputPath = tempDir.copyResourceFileName("set1.txt");
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
    PCollection<Pair<String, Long>> set1Lengths = set1Lines.parallelDo(new StringLengthMapFn(),
        Writables.pairs(Writables.strings(), Writables.longs()));
    PTable<String, Long> set2Counts = pipeline.read(At.textFile(inputPath2, Writables.strings())).count();
    PTables.asPTable(set2Counts.union(set1Lengths)).groupByKey().ungroup()
        .write(At.sequenceFile(output, Writables.strings(), Writables.longs()));
    PipelineResult res = pipeline.done();
    assertEquals(4, res.getStageResults().get(0).getCounterValue("my", "counter"));
  }

  @Test
  public void testMultiWrite() throws Exception {
View Full Code Here
130
131
132
133
134
135
136
137
    expectedPairs.add(Pair.of("c", 1L));
    expectedPairs.add(Pair.of("d", 1L));

    assertEquals(expectedPairs, values);

    pipeline.done();
  }
}
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
    expected.add(Pair.of("a", wrap("a")));
    expected.add(Pair.of("c", wrap("c")));
    expected.add(Pair.of("d", wrap("d")));

    assertEquals(expected, Lists.newArrayList(sorted.materialize()));
    pipeline.done();
  }

  @Test
  public void testAvroReflectSortTable() throws IOException {
    Pipeline pipeline = new SparkPipeline("local", "sort");
View Full Code Here
174
175
176
177
178
179
180
181
182
183
184
    expected.add(Pair.of("a", wrap("a")));
    expected.add(Pair.of("c", wrap("c")));
    expected.add(Pair.of("d", wrap("d")));

    assertEquals(expected, Lists.newArrayList(sorted.materialize()));
    pipeline.done();
  }

  @Test
  public void testAvroSortTable() throws Exception {
    runTable(new SparkPipeline("local", "sort"), AvroTypeFamily.getInstance(), "A");
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
    PCollection<String> set2 = pipeline.read(From.textFile(tempDir.copyResourceFileName("set2.txt")));
    Iterable<Pair<Integer, Long>> cnts = set1.union(set2)
        .parallelDo(new CntFn(), Avros.ints())
        .count().materialize();
    assertEquals(ImmutableList.of(Pair.of(1, 7L)), Lists.newArrayList(cnts));
    pipeline.done();
  }

  private static class CntFn extends MapFn<String, Integer> {
    @Override
    public Integer map(String input) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.