Package org.apache.crunch.impl.mr

Examples of org.apache.crunch.impl.mr.MRPipeline.done()

68
69
70
71
72
73
74
75
76
77
78
        .parallelDo(extractIPResponseSize, Writables.tableOf(Writables.strings(), Writables.longs())).groupByKey()
        .combineValues(longSumCombiner);

    pipeline.writeTextFile(ipAddrResponseSize, args[1]);
    // Execute the pipeline as a MapReduce.
    PipelineResult result = pipeline.done();

    return result.succeeded() ? 0 : 1;
  }

  // Function to parse apache log records
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
    PCollection<Put> resultPut = createPut(result);

    // We write the puts in hbase, in the target table
    pipeline.write(resultPut, new HBaseTarget(TABLE_TARGET));

    pipeline.done();
    return 0;
  }

  /**
   * Put the puts in HBase
View Full Code Here
69
70
71
72
73
74
75
76
        return j.join(input.first(), j.join(input.second()));
      }
    }, ptf.strings()).materialize();
    assertEquals(ImmutableList.of("one,[-5,10],[1,1],[2,-3]", "three,[0,-1]", "two,[1,7],[2,6],[4,5]"),
        ImmutableList.copyOf(lines));
    p.done();
  }
}
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
    assertEquals(3, sortedByAgeList.size());
    assertEquals(ccc10, sortedByAgeList.get(0));
    assertEquals(bbb20, sortedByAgeList.get(1));
    assertEquals(aaa30, sortedByAgeList.get(2));

    pipeline.done();
  }

  private static void writeAvroFile(List<Person> people, File avroFile) throws IOException {

    FileOutputStream outputStream = new FileOutputStream(avroFile);
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
            return Pair.of(p[0], p[1]);
          }
        }, Avros.tableOf(Avros.strings(), Avros.strings()))
        .groupByKey()
        .write(new AvroPathPerKeyTarget(outDir));
    p.done();

    Set<String> names = Sets.newHashSet();
    FileSystem fs = outDir.getFileSystem(tempDir.getDefaultConfiguration());
    for (FileStatus fstat : fs.listStatus(outDir)) {
      names.add(fstat.getPath().getName());
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
          }
        }, Avros.tableOf(Avros.strings(), Avros.strings()))
        .filter(FilterFns.<Pair<String, String>>REJECT_ALL())
        .groupByKey()
        .write(new AvroPathPerKeyTarget(outDir));
    p.done();

    FileSystem fs = outDir.getFileSystem(tempDir.getDefaultConfiguration());
    assertFalse(fs.exists(outDir));
  }
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
    org.apache.crunch.Pair<Person, Integer> firstEntry = Iterables.getFirst(personTable.materialize(), null);

    assertEquals("a", firstEntry.first().getName().toString());
    assertEquals(Integer.valueOf(1), firstEntry.second());

    pipeline.done();

  }

  @Test
  public void testInputFromMapRedKeyValueFile_Specific() throws IOException {
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
    PTableType<Person, Integer> tableType = Avros.keyValueTableOf(Avros.specifics(Person.class), Avros.ints());
    tableType.initialize(tempDir.getDefaultConfiguration());
    org.apache.crunch.Pair<Person, Integer> detachedPair = tableType.getDetachedValue(firstEntry);
    assertEquals(firstEntry, detachedPair);

    pipeline.done();
  }

  @Test
  public void testInputFromMapRedKeyValueFile_Reflect() throws IOException {
    Path keyValuePath = produceMapRedOutputFile();
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
        Avros.keyValueTableOf(Avros.reflects(ReflectedPerson.class), Avros.ints());
    tableType.initialize(tempDir.getDefaultConfiguration());
    org.apache.crunch.Pair<ReflectedPerson, Integer> detachedPair = tableType.getDetachedValue(firstEntry);
    assertEquals(firstEntry, detachedPair);

    pipeline.done();
  }

  /**
   * Produces an Avro file using the org.apache.avro.mapred.* API.
   */
 
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
      }
    }, Writables.tableOf(Writables.writables(IntWritable.class), Writables.writables(Text.class)));
   
    PTable<IntWritable, Text> out = Mapreduce.map(two, TestMapper.class, IntWritable.class, Text.class);
    out.write(To.sequenceFile(tempDir.getPath("temp")));
    PipelineResult res = p.done();
    assertEquals(1, res.getStageResults().size());
    StageResult sr = res.getStageResults().get(0);
    assertEquals(3667, sr.getCounters().findCounter("written", "out").getValue());
  }
 
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.