Package org.apache.crunch

Examples of org.apache.crunch.Target

95
96
97
98
99
100
101
102
103
104
105
106
107
    }
  }

  @Test
  public void testSequentialDo() throws IOException, InterruptedException {
    Target t1 = To.textFile("foo");
    Target t2 = To.textFile("bar");
    Target t3 = To.textFile("baz");
    IncrementingPipelineCallable first = new IncrementingPipelineCallable("first");
    IncrementingPipelineCallable second = new IncrementingPipelineCallable("second");
    IncrementingPipelineCallable third = new IncrementingPipelineCallable("third");
    CrunchControlledJob job1 = createJob(1, ImmutableSet.of(t1));
    CrunchControlledJob job2 = createJob(2, ImmutableSet.of(t2));
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
        dotfileWriter.formatPCollectionNodeDeclaration(inputCollection, jobPrototype));
  }

  @Test
  public void testFormatTargetNodeDeclaration() {
    Target target = mock(Target.class);
    when(target.toString()).thenReturn("target/path");

    assertEquals("\"target/path\" [label=\"target/path\" shape=folder];",
        dotfileWriter.formatTargetNodeDeclaration(target));
  }
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
    Pipeline pipeline = new MRPipeline(TrevniKeyPipelineIT.class, tmpDir.getDefaultConfiguration());
    PCollection<Person> genericCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(),
        Avros.records(Person.class)));
    File outputFile = tmpDir.getFile("output");
    Target trevniFile = new TrevniKeyTarget(outputFile.getAbsolutePath());
    pipeline.write(genericCollection, trevniFile);
    pipeline.run();

    Person person = genericCollection.materialize().iterator().next();
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
    Pipeline pipeline = new MRPipeline(TrevniKeyPipelineIT.class, tmpDir.getDefaultConfiguration());
    PCollection<Person> genericCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(),
        Avros.records(Person.class)));
    File outputFile = tmpDir.getFile("output");
    Target trevniFile = new TrevniKeyTarget(outputFile.getAbsolutePath());
    pipeline.write(genericCollection, trevniFile);
    pipeline.run();

    Person person = genericCollection.materialize().iterator().next();
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
    Pipeline pipeline = new MRPipeline(AvroFileSourceTargetIT.class, tmpDir.getDefaultConfiguration());
    PCollection<Person> genericCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(),
        Avros.records(Person.class)));
    File outputFile = tmpDir.getFile("output");
    Target textFile = To.textFile(outputFile.getAbsolutePath());
    pipeline.write(genericCollection, textFile);
    pipeline.run();
    Person person = genericCollection.materialize().iterator().next();
    String outputString = FileUtils.readFileToString(new File(outputFile, "part-m-00000"));
    assertTrue(outputString.contains(person.toString()));
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
    Pipeline pipeline = new MRPipeline(AvroParquetPipelineIT.class, tmpDir.getDefaultConfiguration());
    PCollection<Person> genericCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(),
        Avros.records(Person.class)));
    File outputFile = tmpDir.getFile("output");
    Target parquetFileTarget = new AvroParquetFileTarget(outputFile.getAbsolutePath());
    pipeline.write(genericCollection, parquetFileTarget);
    pipeline.run();

    Person person = genericCollection.materialize().iterator().next();
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
    Pipeline pipeline = new MRPipeline(AvroParquetPipelineIT.class, tmpDir.getDefaultConfiguration());
    PCollection<Person> genericCollection = pipeline.read(
        new AvroParquetFileSource<Person>(new Path(avroFile.getAbsolutePath()), Avros.records(Person.class)));
    File outputFile = tmpDir.getFile("output");
    Target parquetFileTarget = new AvroParquetFileTarget(outputFile.getAbsolutePath());
    pipeline.write(genericCollection, parquetFileTarget);
    pipeline.run();

    Person person = genericCollection.materialize().iterator().next();
View Full Code Here
208
209
210
211
212
213
214
215
216
217
218
    Pipeline pipeline = new MRPipeline(AvroParquetPipelineIT.class, tmpDir.getDefaultConfiguration());
    PCollection<Person> genericCollection = pipeline.read(At.avroFile(avroFile.getAbsolutePath(),
        Avros.records(Person.class)));
    File outputFile = tmpDir.getFile("output");
    Target parquetFileTarget = new AvroParquetFileTarget(outputFile.getAbsolutePath());
    pipeline.write(genericCollection, parquetFileTarget);
    pipeline.run();

    Person person = genericCollection.materialize().iterator().next();
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
        AvroParquetFileSource.builder(Person.class)
            .includeField("age")
            .build(new Path(avroFile.getAbsolutePath())));

    File outputFile = tmpDir.getFile("output");
    Target avroFile = To.avroFile(outputFile.getAbsolutePath());
    genericCollection.write(avroFile);
    pipeline.done();
   
    Pipeline pipeline2 = new MRPipeline(AvroParquetFileSourceTargetIT.class,
        tmpDir.getDefaultConfiguration());
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
        .build(new Path(avroFile.getAbsolutePath()));
    Pipeline pipeline = new MRPipeline(AvroParquetFileSourceTargetIT.class, tmpDir.getDefaultConfiguration());
    PCollection<GenericRecord> genericCollection = pipeline.read(src);

    File outputFile = tmpDir.getFile("output");
    Target avroFile = To.avroFile(outputFile.getAbsolutePath());
    genericCollection.write(avroFile);
    pipeline.done();

    Pipeline pipeline2 = new MRPipeline(AvroParquetFileSourceTargetIT.class,
        tmpDir.getDefaultConfiguration());
View Full Code Here
TOP

Related Classes of org.apache.crunch.Target

  • org.apache.crunch.hadoop.mapreduce.lib.jobcontrol.CrunchJobControlTest
  • org.apache.crunch.impl.mem.MemPipelineFileReadingWritingIT
  • org.apache.crunch.impl.mem.MemPipelineUTF8IT
  • org.apache.crunch.impl.mr.plan.DotfileWriterTest
  • org.apache.crunch.impl.mr.plan.MSCRPlanner
  • org.apache.crunch.io.avro.AvroPipelineIT
  • org.apache.crunch.io.avro.trevni.TrevniKeyPipelineIT
  • org.apache.crunch.io.parquet.AvroParquetFileSourceTargetIT
  • org.apache.crunch.io.parquet.AvroParquetPipelineIT
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.