Package com.cloudera.crunch

Examples of com.cloudera.crunch.Pipeline


public class SortCrunchTest implements Serializable {
 
  @Test
  public void test() throws IOException {
    Pipeline pipeline = new MRPipeline(SortCrunchTest.class);
    PCollection<String> records = pipeline.readTextFile("sort/A");
   
    PCollection<Pair<Integer, Integer>> pairs = records.parallelDo(new DoFn<String, Pair<Integer, Integer>>() {
      @Override
      public void process(String input, Emitter<Pair<Integer, Integer>> emitter) {
        Iterator<String> split = Splitter.on('\t').split(input).iterator();
        String l = split.next();
        String r = split.next();
        emitter.emit(Pair.of(Integer.parseInt(l), Integer.parseInt(r)));
      }
    }, pairs(ints(), ints()));
   
    PCollection<Pair<Integer, Integer>> sorted = Sort.sortPairs(pairs, by(1, ASCENDING), by(2, DESCENDING));
   
    pipeline.writeTextFile(sorted, "output-sorted");
    pipeline.run();
  }
View Full Code Here


 
  private static final int MISSING = 9999;
 
  @Test
  public void test() throws IOException {
    Pipeline pipeline = new MRPipeline(MaxTemperatureCrunchTest.class);
    PCollection<String> records = pipeline.readTextFile("input");
   
    PTable<String, Integer> maxTemps = records
      .parallelDo(toYearTempPairsFn(), tableOf(strings(), ints()))
      .groupByKey()
      .combineValues(CombineFn.<String> MAX_INTS());
   
    pipeline.writeTextFile(maxTemps, "output");
    pipeline.run();
  }
View Full Code Here

public class JoinCrunchTest implements Serializable {
 
  @Test
  public void test() throws IOException {
    Pipeline pipeline = new MRPipeline(JoinCrunchTest.class);
    PCollection<String> a = pipeline.readTextFile("join/A");
    PCollection<String> b = pipeline.readTextFile("join/B");
   
    PTable<String, String> aTable = a.parallelDo(new DoFn<String, Pair<String, String>>() {
    @Override
    public void process(String input, Emitter<Pair<String, String>> emitter) {
      Iterator<String> split = Splitter.on('\t').split(input).iterator();
      emitter.emit(Pair.of(split.next(), split.next()));
    }
  }, tableOf(strings(),strings()));

    PTable<String, String> bTable = b.parallelDo(new DoFn<String, Pair<String, String>>() {
    @Override
    public void process(String input, Emitter<Pair<String, String>> emitter) {
      Iterator<String> split = Splitter.on('\t').split(input).iterator();
      String l = split.next();
      String r = split.next();
      emitter.emit(Pair.of(r, l));
    }
  }, tableOf(strings(),strings()));
   
    PTable<String, Pair<String, String>> join = Join.join(aTable, bTable);
   
    pipeline.writeTextFile(join, "output-joined");
    pipeline.run();
  }
View Full Code Here

public class CogroupCrunchTest implements Serializable {
 
  @Test
  public void test() throws IOException {
    Pipeline pipeline = new MRPipeline(CogroupCrunchTest.class);
    PCollection<String> a = pipeline.readTextFile("join/A");
    PCollection<String> b = pipeline.readTextFile("join/B");
   
    PTable<String, String> aTable = a.parallelDo(new DoFn<String, Pair<String, String>>() {
    @Override
    public void process(String input, Emitter<Pair<String, String>> emitter) {
      Iterator<String> split = Splitter.on('\t').split(input).iterator();
      emitter.emit(Pair.of(split.next(), split.next()));
    }
  }, tableOf(strings(),strings()));

    PTable<String, String> bTable = b.parallelDo(new DoFn<String, Pair<String, String>>() {
    @Override
    public void process(String input, Emitter<Pair<String, String>> emitter) {
      Iterator<String> split = Splitter.on('\t').split(input).iterator();
      String l = split.next();
      String r = split.next();
      emitter.emit(Pair.of(r, l));
    }
  }, tableOf(strings(),strings()));
   
    PTable<String, Pair<Collection<String>, Collection<String>>> cogroup = Cogroup.cogroup(aTable, bTable);
   
    pipeline.writeTextFile(cogroup, "output-cogrouped");
    pipeline.run();
  }
View Full Code Here

TOP

Related Classes of com.cloudera.crunch.Pipeline

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.