Package org.apache.crunch.impl.mr

Examples of org.apache.crunch.impl.mr.MRPipeline.done()

73
74
75
76
77
78
79
80
81
82
83
    File outputDirA = tmpDir.getFile("output_a");
    File outputDirB = tmpDir.getFile("output_b");
   
    pipeline.writeTextFile(ungroupedTableA, outputDirA.getAbsolutePath());
    pipeline.writeTextFile(ungroupedTableB, outputDirB.getAbsolutePath());
    PipelineResult result = pipeline.done();
    for(StageResult stageResult : result.getStageResults()){
      assertTrue(stageResult.getStageName().length() > 1);
      assertTrue(stageResult.getStageId().length() > 1);
    }
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
    PCollection<String> lines = pipeline.readTextFile(tmpDir.copyResourceFileName("set1.txt"));
    pipeline.write(
        lines.parallelDo(IdentityFn.<String>getInstance(), Writables.strings()),
        To.textFile(tmpDir.getFile("output").getAbsolutePath()));
    pipeline.done();

    File[] files = dotfileDir.listFiles((FileFilter)new SuffixFileFilter(".dot"));
    assertEquals(1, files.length);
    String fileName = files[0].getName();
    String fileNamePrefix = URLEncoder.encode(pipeline.getName(), "UTF-8");
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
            .build(new Path(avroFile.getAbsolutePath())));

    File outputFile = tmpDir.getFile("output");
    Target avroFile = To.avroFile(outputFile.getAbsolutePath());
    genericCollection.write(avroFile);
    pipeline.done();
   
    Pipeline pipeline2 = new MRPipeline(AvroParquetFileSourceTargetIT.class,
        tmpDir.getDefaultConfiguration());
    PCollection<Person> ageOnly = pipeline2.read(
        new AvroFileSource<Person>(new Path(outputFile.getAbsolutePath()), Avros.specifics(Person.class)));
View Full Code Here
167
168
169
170
171
172
173
174
175
176
177
    PCollection<GenericRecord> genericCollection = pipeline.read(src);

    File outputFile = tmpDir.getFile("output");
    Target avroFile = To.avroFile(outputFile.getAbsolutePath());
    genericCollection.write(avroFile);
    pipeline.done();

    Pipeline pipeline2 = new MRPipeline(AvroParquetFileSourceTargetIT.class,
        tmpDir.getDefaultConfiguration());
    PCollection<Record> ageOnly = pipeline2.read(
        new AvroFileSource<Record>(new Path(outputFile.getAbsolutePath()), Avros.generics(src.getProjectedSchema())));
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
      @Override
      public boolean accept(Person input) {
        return input != null;
      }
    }).write(avroFile);
    pipeline.done();

    Pipeline pipeline2 = new MRPipeline(AvroParquetFileSourceTargetIT.class,
        tmpDir.getDefaultConfiguration());
    PCollection<Person> ageOnly = pipeline2.read(
        new AvroFileSource<Person>(new Path(outputFile.getAbsolutePath()), Avros.specifics(Person.class)));
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
    String outputA = tmpDir.getFileName("stringsA");
    String outputB = tmpDir.getFileName("stringsB");

    pipeline.writeTextFile(stringsA, outputA);
    pipeline.writeTextFile(stringsB, outputB);
    PipelineResult pipelineResult = pipeline.done();

    // Make sure fusing did actually occur
    assertEquals(1, pipelineResult.getStageResults().size());

    checkFileContents(outputA, Lists.newArrayList("cA", "dA", "aA"));
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
  public void testWritables() throws Exception {
    Pipeline pipeline = new MRPipeline(AggregateIT.class, tmpDir.getDefaultConfiguration());
    String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
    PCollection<String> shakes = pipeline.readTextFile(shakesInputPath);
    runMinMax(shakes, WritableTypeFamily.getInstance());
    pipeline.done();
  }

  @Test
  public void testAvro() throws Exception {
    Pipeline pipeline = new MRPipeline(AggregateIT.class, tmpDir.getDefaultConfiguration());
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
  public void testAvro() throws Exception {
    Pipeline pipeline = new MRPipeline(AggregateIT.class, tmpDir.getDefaultConfiguration());
    String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt");
    PCollection<String> shakes = pipeline.readTextFile(shakesInputPath);
    runMinMax(shakes, AvroTypeFamily.getInstance());
    pipeline.done();
  }

  @Test
  public void testInMemoryAvro() throws Exception {
    PCollection<String> someText = MemPipeline.collectionOf("first line", "second line", "third line");
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
        expectedSize = 2;
      } else if ("www.C.com".equals(key) || "www.D.com".equals(key) || "www.E.com".equals(key)) {
        expectedSize = 1;
      }
      assertEquals("Checking key = " + key, expectedSize, e.second().size());
      p.done();
    }
  }

  @Test
  public void testTopN() throws Exception {
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
    File outputDirA = tmpDir.getFile("output_a");
    File outputDirB = tmpDir.getFile("output_b");
   
    pipeline.writeTextFile(ungroupedTableA, outputDirA.getAbsolutePath());
    pipeline.writeTextFile(ungroupedTableB, outputDirB.getAbsolutePath());
    pipeline.done();

    // Verify that output from a single PGroupedTable can be sent to multiple collections
    assertTrue(new File(outputDirA, "part-r-00000").exists());
    assertTrue(new File(outputDirB, "part-r-00000").exists());
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.