Package org.apache.crunch.impl.mr

Examples of org.apache.crunch.impl.mr.MRPipeline.done()

75
76
77
78
79
80
81
82
83
84
85
        Writables.tableOf(Writables.strings(), Writables.doubles()));

    // write the result to a text file
    pipeline.writeTextFile(avgs, args[1]);
    // Execute the pipeline as a MapReduce.
    pipeline.done();
    return 0;
  }

  // Function to calculate the average response size for a given ip address
  //
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
        .parallelDo(extractIPResponseSize, Writables.tableOf(Writables.strings(), Writables.longs())).groupByKey()
        .combineValues(longSumCombiner);

    pipeline.writeTextFile(ipAddrResponseSize, args[1]);
    // Execute the pipeline as a MapReduce.
    pipeline.done();
    return 0;
  }

  // Function to parse apache log records
  // Given a standard apache log line, extract the ip address and
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
    PTable<String, Long> counts = words.count();

    // Instruct the pipeline to write the resulting counts to a text file.
    pipeline.writeTextFile(counts, args[2]);
    // Execute the pipeline as a MapReduce.
    pipeline.done();
    return 0;
  }

  public static void main(String[] args) throws Exception {
    ToolRunner.run(new Configuration(), new WordCount(), args);
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
    PCollection<Put> resultPut = createPut(result);

    // We write the puts in hbase, in the target table
    pipeline.write(resultPut, new HBaseTarget(TABLE_TARGET));

    pipeline.done();
    return 0;
  }

  /**
   * Put the puts in HBase
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
      }

    }, Writables.strings());

    List<String> nameList = Lists.newArrayList(names.materialize());
    pipeline.done();

    assertEquals(2, nameList.size());
    assertEquals(Sets.newHashSet("Hello", "World"), Sets.newHashSet(nameList));

  }
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
        .parallelDo(new PreProcFn(), tableOf(ints(), pairs(ints(), pids)))
        .groupByKey()
        .parallelDo(new PostProcFn(), strings())
        .materialize();
    assertEquals(65, Iterables.size(out));
    p.done();
  }
 
  private static class PreProcFn extends MapFn<String, Pair<Integer, Pair<Integer, PID>>> {
    private int counter = 0;
    @Override
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
    File outputDirA = tmpDir.getFile("output_a");
    File outputDirB = tmpDir.getFile("output_b");
   
    pipeline.writeTextFile(ungroupedTableA, outputDirA.getAbsolutePath());
    pipeline.writeTextFile(ungroupedTableB, outputDirB.getAbsolutePath());
    pipeline.done();

    // Verify that output from a single PGroupedTable can be sent to multiple collections
    assertTrue(new File(outputDirA, "part-r-00000").exists());
    assertTrue(new File(outputDirB, "part-r-00000").exists());
  }
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
    File cso = tmpDir.getFile("cleanShakes");
    cleanShakes.write(To.textFile(cso.getAbsolutePath()));
   
    File wc = tmpDir.getFile("wordCounts");
    cleanShakes.parallelDo(SPLIT, Avros.strings()).count().write(To.textFile(wc.getAbsolutePath()));
    pipeline.done();
   
    File cleanFile = new File(cso, "part-m-00000");
    List<String> lines = Files.readLines(cleanFile, Charset.defaultCharset());
    assertEquals(LINES_IN_SHAKES, lines.size());
  }
View Full Code Here
92
93
94
95
96
97
98
99
100
101
      if (v.first().equals("k") && v.second().get("n") == 8L) {
        passed = true;
        break;
      }
    }
    pipeline.done();

    assertThat(passed, is(true));
  }
}
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
    String outputA = tmpDir.getFileName("stringsA");
    String outputB = tmpDir.getFileName("stringsB");

    pipeline.writeTextFile(stringsA, outputA);
    pipeline.writeTextFile(stringsB, outputB);
    PipelineResult pipelineResult = pipeline.done();

    // Make sure fusing did actually occur
    assertEquals(1, pipelineResult.getStageResults().size());

    checkFileContents(outputA, Lists.newArrayList("cA", "dA", "aA"));
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.