Package org.apache.crunch.impl.mr

Examples of org.apache.crunch.impl.mr.MRPipeline.done()

69
70
71
72
73
74
75
76
77
78
79
  public void testAvro() throws Exception {
    Pipeline pipeline = new MRPipeline(AggregateIT.class, tmpDir.getDefaultConfiguration());
    String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
    PCollection<String> shakes = pipeline.readTextFile(shakesInputPath);
    runMinMax(shakes, AvroTypeFamily.getInstance());
    pipeline.done();
  }

  @Test
  public void testInMemoryAvro() throws Exception {
    PCollection<String> someText = MemPipeline.collectionOf("first line", "second line", "third line");
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
        expectedSize = 2;
      } else if ("www.C.com".equals(key) || "www.D.com".equals(key) || "www.E.com".equals(key)) {
        expectedSize = 1;
      }
      assertEquals("Checking key = " + key, expectedSize, e.second().size());
      p.done();
    }
  }

  @Test
  public void testTopN() throws Exception {
View Full Code Here
205
206
207
208
209
210
211
212
213
214
215
    String outputA = tmpDir.getFileName("stringsA");
    String outputB = tmpDir.getFileName("stringsB");

    pipeline.writeTextFile(stringsA, outputA);
    pipeline.writeTextFile(stringsB, outputB);
    PipelineResult pipelineResult = pipeline.done();

    // Make sure fusing did actually occur
    assertEquals(1, pipelineResult.getStageResults().size());

    checkFileContents(outputA, Lists.newArrayList("cA", "dA", "aA"));
View Full Code Here
42
43
44
45
46
47
48
49
        return new Pair<String, Person>("first name", null);
      }
    }, Avros.tableOf(Avros.strings(), Avros.records(Person.class)))
        .write(new AvroFileTarget(outDir), Target.WriteMode.APPEND);

    p.done();
  }
}
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
          }
        }, Avros.reflects(StringWrapper.class));

    List<StringWrapper> stringWrappers = Lists.newArrayList(stringWrapperCollection.materialize());

    pipeline.done();

    assertEquals(Lists.newArrayList(new StringWrapper("b"), new StringWrapper("c"), new StringWrapper("a"),
        new StringWrapper("e")), stringWrappers);

  }
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
    List<Pair<String, Long>> expected = Lists.newArrayList(Pair.of("a", 1L), Pair.of("b", 1L), Pair.of("c", 1L),
        Pair.of("e", 1L));
    Collections.sort(materialized);

    assertEquals(expected, materialized);
    pipeline.done();
  }

  private static PType<String> STRING_PTYPE = Avros.derived(String.class,
      new MapFn<StringWrapper, String>() { public String map(StringWrapper in) { return in.getValue(); }},
      new MapFn<String, StringWrapper>() { public StringWrapper map(String out) { return new StringWrapper(out); }},
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
  public void testDerivedReflection() throws Exception {
    Pipeline pipeline = new MRPipeline(AvroReflectIT.class, tmpDir.getDefaultConfiguration());
    PCollection<String> stringWrapperCollection = pipeline.readTextFile(tmpDir.copyResourceFileName("set1.txt"))
        .parallelDo(IdentityFn.<String>getInstance(), STRING_PTYPE);
    List<String> strings = Lists.newArrayList(stringWrapperCollection.materialize());
    pipeline.done();
    assertEquals(Lists.newArrayList("b", "c", "a", "e"), strings);
  }

  @Test
  public void testWrappedDerivedReflection() throws Exception {
View Full Code Here
133
134
135
136
137
138
139
140
141
142
          public Pair<Long, String> map(String input) {
            return Pair.of(1L, input);
          }
        }, Avros.pairs(Avros.longs(), STRING_PTYPE));
    List<Pair<Long, String>> pairs = Lists.newArrayList(stringWrapperCollection.materialize());
    pipeline.done();
    assertEquals(pairs.size(), 4);
    assertEquals(Pair.of(1L, "a"), pairs.get(2));
  }
}
View Full Code Here
116
117
118
119
120
121
122
123
124
      }
    }, Avros.tableOf(Avros.longs(), Avros.reflects(StringWrapper.class)))
        .groupByKey()
        .ungroup();
    List<Pair<Long, StringWrapper>> ret = Lists.newArrayList(pt.materialize());
    pipeline.done();
    assertEquals(1, ret.size());
  }
}
View Full Code Here
137
138
139
140
141
142
143
144
            }
          }
        });

    pipeline.writeTextFile(result, tmpDir.getFileName("unused"));
    Assert.assertTrue("Should succeed", pipeline.done().succeeded());
  }
}
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.