Package org.apache.crunch

Examples of org.apache.crunch.Target

196
197
198
199
200
201
202
203
204
205
206
            .includeField("age")
            .filterClass(RejectAllFilter.class)
            .build(new Path(avroFile.getAbsolutePath())));

    File outputFile = tmpDir.getFile("output");
    Target avroFile = To.avroFile(outputFile.getAbsolutePath());
    genericCollection.filter(new FilterFn<Person>() {
      @Override
      public boolean accept(Person input) {
        return input != null;
      }
View Full Code Here
418
419
420
421
422
423
424
425
426
427
428
    if (!outputs.containsKey(splitTarget)) {
      outputs.put(splitTarget, Sets.<Target> newHashSet());
    }

    SourceTarget srcTarget = null;
    Target targetToReplace = null;
    for (Target t : outputs.get(splitTarget)) {
      if (t instanceof SourceTarget) {
        srcTarget = (SourceTarget<?>) t;
        break;
      } else {
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
  @Test
  public void testMemPipelineWriteSequenceFile_PCollection() throws IOException {
    // write
    PCollection<String> collection = MemPipeline.typedCollectionOf(Writables.strings(), EXPECTED_COLLECTION);
    final Target target = To.sequenceFile(outputFile.toString());
    MemPipeline.getInstance().write(collection, target);

    // read
    final SequenceFile.Reader reader = new Reader(FileSystem.getLocal(
      baseTmpDir.getDefaultConfiguration()), new Path(outputFile.toString()),
View Full Code Here
147
148
149
150
151
152
153
154
155
156
157
    // write
    final MemTable<Integer, String> collection = new MemTable<Integer, String>(EXPECTED_TABLE, //
        Writables.tableOf(
          Writables.ints(),
          Writables.strings()), "test input");
    final Target target = To.sequenceFile(outputFile.toString());
    MemPipeline.getInstance().write(collection, target);

    // read
    final SequenceFile.Reader reader = new Reader(FileSystem.getLocal(baseTmpDir
        .getDefaultConfiguration()), new Path(outputFile.toString()),
View Full Code Here
374
375
376
377
378
379
380
381
382
383
384
    if (!outputs.containsKey(splitTarget)) {
      outputs.put(splitTarget, Sets.<Target> newHashSet());
    }

    SourceTarget srcTarget = null;
    Target targetToReplace = null;
    for (Target t : outputs.get(splitTarget)) {
      if (t instanceof SourceTarget) {
        srcTarget = (SourceTarget<?>) t;
        break;
      } else {
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
        dotfileWriter.formatPCollectionNodeDeclaration(inputCollection, jobPrototype));
  }

  @Test
  public void testFormatTargetNodeDeclaration() {
    Target target = mock(Target.class);
    when(target.toString()).thenReturn("target/path");

    assertEquals("\"target/path\" [label=\"target/path\" shape=folder];",
        dotfileWriter.formatTargetNodeDeclaration(target));
  }
View Full Code Here
296
297
298
299
300
301
302
303
304
305
306
    if (!outputs.containsKey(splitTarget)) {
      outputs.put(splitTarget, Sets.<Target> newHashSet());
    }

    SourceTarget srcTarget = null;
    Target targetToReplace = null;
    for (Target t : outputs.get(splitTarget)) {
      if (t instanceof SourceTarget) {
        srcTarget = (SourceTarget<?>) t;
        break;
      } else {
View Full Code Here
343
344
345
346
347
348
349
350
351
352
353
    if (!outputs.containsKey(splitTarget)) {
      outputs.put(splitTarget, Sets.<Target> newHashSet());
    }

    SourceTarget srcTarget = null;
    Target targetToReplace = null;
    for (Target t : outputs.get(splitTarget)) {
      if (t instanceof SourceTarget) {
        srcTarget = (SourceTarget<?>) t;
        break;
      } else {
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
    Pipeline pipeline = new MRPipeline(AvroFileSourceTargetIT.class, tmpDir.getDefaultConfiguration());
    PCollection<Person> genericCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(),
        Avros.records(Person.class)));
    File outputFile = tmpDir.getFile("output");
    Target textFile = To.textFile(outputFile.getAbsolutePath());
    pipeline.write(genericCollection, textFile);
    pipeline.run();
    Person person = genericCollection.materialize().iterator().next();
    String outputString = FileUtils.readFileToString(new File(outputFile, "part-m-00000"));
    assertTrue(outputString.contains(person.toString()));
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
        dotfileWriter.formatPCollectionNodeDeclaration(inputCollection, jobPrototype));
  }

  @Test
  public void testFormatTargetNodeDeclaration() {
    Target target = mock(Target.class);
    when(target.toString()).thenReturn("target/path");

    assertEquals("\"target/path\" [label=\"target/path\" shape=folder];",
        dotfileWriter.formatTargetNodeDeclaration(target));
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.Target

  • org.apache.crunch.hadoop.mapreduce.lib.jobcontrol.CrunchJobControlTest
  • org.apache.crunch.impl.mem.MemPipelineFileReadingWritingIT
  • org.apache.crunch.impl.mem.MemPipelineUTF8IT
  • org.apache.crunch.impl.mr.plan.DotfileWriterTest
  • org.apache.crunch.impl.mr.plan.MSCRPlanner
  • org.apache.crunch.io.avro.AvroPipelineIT
  • org.apache.crunch.io.avro.trevni.TrevniKeyPipelineIT
  • org.apache.crunch.io.parquet.AvroParquetFileSourceTargetIT
  • org.apache.crunch.io.parquet.AvroParquetPipelineIT
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.