7879808182838485868788
Writables.tableOf(Writables.strings(), Writables.doubles())); // write the result to a text file pipeline.writeTextFile(avgs, args[1]); // Execute the pipeline as a MapReduce. PipelineResult result = pipeline.done(); return result.succeeded() ? 0 : 1; } // Function to calculate the average response size for a given ip address
6970717273747576777879
.parallelDo(extractIPResponseSize, Writables.tableOf(Writables.strings(), Writables.longs())).groupByKey() .combineValues(agg); pipeline.writeTextFile(ipAddrResponseSize, args[1]); // Execute the pipeline as a MapReduce. PipelineResult result = pipeline.done(); return result.succeeded() ? 0 : 1; } // Function to parse apache log records
6566676869707172737475
PTable<String, Long> counts = words.count(); // Instruct the pipeline to write the resulting counts to a text file. pipeline.writeTextFile(counts, args[1]); // Execute the pipeline as a MapReduce. PipelineResult result = pipeline.done(); return result.succeeded() ? 0 : 1; } public static void main(String[] args) throws Exception {
5859606162636465
return j.join(input.first(), j.join(input.second())); } }, strings()).materialize(); assertEquals(ImmutableList.of("one,[-5,10],[1,1],[2,-3]", "three,[0,-1]", "two,[1,7],[2,6],[4,5]"), ImmutableList.copyOf(lines)); p.done(); } }
132133134135136137138139140141142
PCollection<Put> resultPut = createPut(result); // We write the puts in hbase, in the target table pipeline.write(resultPut, new HBaseTarget(TABLE_TARGET)); pipeline.done(); return 0; } /** * Put the puts in HBase
113114115116117118119120121122123
assertEquals(3, sortedByAgeList.size()); assertEquals(ccc10, sortedByAgeList.get(0)); assertEquals(bbb20, sortedByAgeList.get(1)); assertEquals(aaa30, sortedByAgeList.get(2)); pipeline.done(); } private void writeAvroFile(List<Person> people, File avroFile) throws IOException { FileOutputStream outputStream = new FileOutputStream(avroFile);
147148149150151152153154155156157
emitter.emit(sb.toString()); } }, Writables.strings()).write(To.textFile(args[1])); // Execute the pipeline as a MapReduce. return pipeline.done().succeeded() ? 0 : 1; } public static void main(String[] args) throws Exception { int exitCode = -1; try {
76777879808182838485
// Instruct the pipeline to write the resulting counts to a text file. pipeline.writeTextFile(counts, outputPath); // Execute the pipeline as a MapReduce. PipelineResult result = pipeline.done(); return result.succeeded() ? 0 : 1; } }
5758596061626364656667
public void testWritables() throws Exception { Pipeline pipeline = new MRPipeline(AggregateIT.class, tmpDir.getDefaultConfiguration()); String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt"); PCollection<String> shakes = pipeline.readTextFile(shakesInputPath); runMinMax(shakes, WritableTypeFamily.getInstance()); pipeline.done(); } @Test public void testAvro() throws Exception { Pipeline pipeline = new MRPipeline(AggregateIT.class, tmpDir.getDefaultConfiguration());
6667686970717273747576
public void testAvro() throws Exception { Pipeline pipeline = new MRPipeline(AggregateIT.class, tmpDir.getDefaultConfiguration()); String shakesInputPath = tmpDir.copyResourceFileName("shakes.txt"); PCollection<String> shakes = pipeline.readTextFile(shakesInputPath); runMinMax(shakes, AvroTypeFamily.getInstance()); pipeline.done(); } @Test public void testInMemoryAvro() throws Exception { PCollection<String> someText = MemPipeline.collectionOf("first line", "second line", "third line");