Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.NullWritable

170
171
172
173
174
175
176
177
178
179
180
   
    AvroRecordReader<Pair<Integer, Long>> recordReader = new AvroRecordReader<Pair<Integer, Long>>(job, fileSplit);
   
    AvroWrapper<Pair<Integer, Long>> inputPair = new AvroWrapper<Pair<Integer, Long>>(null);
    NullWritable ignore = NullWritable.get();
   
    long sumOfCounts = 0;
    long numOfCounts = 0;
    while(recordReader.next(inputPair, ignore)) {
      Assert.assertEquals((Integer)inputPair.datum().get(0), defaultRank);
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
   
    AvroRecordReader<Pair<Integer, Long>> recordReader = new AvroRecordReader<Pair<Integer, Long>>(job, fileSplit);
   
    AvroWrapper<Pair<Integer, Long>> inputPair = new AvroWrapper<Pair<Integer, Long>>(null);
    NullWritable ignore = NullWritable.get();
   
    long sumOfCounts = 0;
    long numOfCounts = 0;
    while(recordReader.next(inputPair, ignore)) {
      Assert.assertEquals((Integer)inputPair.datum().get(0), defaultRank);
View Full Code Here
252
253
254
255
256
257
258
259
260
261
262
    String dir = System.getProperty("test.dir", ".") + "/mapred";
    Path inputPath = new Path(dir + "/out" + "/myavro1-r-00000.avro");
    FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath);
    FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job);
    AvroWrapper<Utf8> inputPair = new AvroWrapper<Utf8>(null);
    NullWritable ignore = NullWritable.get();
    AvroRecordReader<Utf8> recordReader = new AvroRecordReader<Utf8>(job, fileSplit);
    long sumOfCounts = 0;
    long numOfCounts = 0;
    while(recordReader.next(inputPair, ignore)) {
        sumOfCounts += Long.parseLong(inputPair.datum().toString().split(":")[2].replace("}","").trim());
View Full Code Here
280
281
282
283
284
285
286
287
288
289
290
    String dir = System.getProperty("test.dir", ".") + "/mapred";
    Path inputPath = new Path(dir + "/out" + "/testavrofile1-r-00000.avro");
    FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath);
    FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job);
    AvroWrapper<Utf8> inputPair = new AvroWrapper<Utf8>(null);
    NullWritable ignore = NullWritable.get();
    AvroRecordReader<Utf8> recordReader = new AvroRecordReader<Utf8>(job, fileSplit);
    long sumOfCounts = 0;
    long numOfCounts = 0;
    while(recordReader.next(inputPair, ignore)) {
        sumOfCounts += Long.parseLong(inputPair.datum().toString().split(":")[2].replace("}","").trim());
View Full Code Here
333
334
335
336
337
338
339
340
341
342
343
    Path inputPath = new Path(dir + "/out" + "/myavro2-m-00000.avro");
    FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath);
    FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job);
    AvroRecordReader<Utf8> recordReader_new = new AvroRecordReader<Utf8>(job, fileSplit);
    AvroWrapper<Utf8> inputPair_new = new AvroWrapper<Utf8>(null);
    NullWritable ignore = NullWritable.get();
    long testl=0;
     while(recordReader_new.next(inputPair_new, ignore)) {
       testl=Long.parseLong(inputPair_new.datum().toString().split(":")[2].replace("}","").trim());
       Assert.assertEquals(onel,testl);
    }
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
    assertEquals("Progress should be zero before any records are read",
        0.0f, recordReader.getProgress(), 0.0f);

    // Some variables to hold the records.
    AvroKey<CharSequence> key;
    NullWritable value;

    // Read the first record.
    assertTrue("Expected at least one record", recordReader.nextKeyValue());
    key = recordReader.getCurrentKey();
    value = recordReader.getCurrentValue();
View Full Code Here
959
960
961
962
963
964
965
966
967
968
969
970
    }

    @Override
    public DataInputBuffer getValue() throws IOException {
      ByteArrayOutputStream dt = new ByteArrayOutputStream();
      NullWritable key = NullWritable.get();
      key.write(new DataOutputStream(dt));
      DataInputBuffer result = new DataInputBuffer();
      byte[] b = dt.toByteArray();
      result.reset(b, 0, b.length);
      return result;
    }
View Full Code Here
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
  TextOutputFormat<?, ?> theOutputFormat = new TextOutputFormat();
  RecordWriter theRecordWriter = theOutputFormat
      .getRecordWriter(tContext);
 
  NullWritable nullWritable = NullWritable.get();
  try {
    theRecordWriter.write(key2, val2);
    theRecordWriter.write(null, nullWritable);
    theRecordWriter.write(null, val2);
    theRecordWriter.write(nullWritable, val1);
View Full Code Here
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
   
    TextOutputFormat<?, ?> theOutputFormat = new TextOutputFormat();
    RecordWriter theRecordWriter = theOutputFormat
        .getRecordWriter(tContext);
   
    NullWritable nullWritable = NullWritable.get();
    try {
      theRecordWriter.write(key1, val1);
      theRecordWriter.write(null, nullWritable);
      theRecordWriter.write(null, val1);
      theRecordWriter.write(nullWritable, val2);
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
  public static Vector retrieveTimesSquaredOutputVector(JobConf conf) throws IOException {
    Path outputPath = FileOutputFormat.getOutputPath(conf);
    FileSystem fs = FileSystem.get(conf);
    Path outputFile = new Path(outputPath, "part-00000");
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, outputFile, conf);
    NullWritable n = NullWritable.get();
    VectorWritable v = new VectorWritable();
    reader.next(n,v);
    Vector vector = v.get();
    reader.close();
    fs.deleteOnExit(outputFile);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.NullWritable

  • bulkimport.BulkImportJobExample$VerboseInputSampler
  • ca.ulaval.ift.graal.pagerank.mapreduce.TopKMapper
  • cc.bf.ReadNgram
  • com.alexholmes.hadooputils.combine.avro.mapred.CombineAvroInputFormatTest
  • com.asakusafw.testtools.TestDataHolder
  • com.ebay.erl.mobius.core.datajoin.EvenlyPartitioner
  • com.facebook.hive.orc.TestInputOutputFormat
  • com.thinkaurelius.faunus.FaunusVertexTest
  • com.twitter.maple.tap.MemorySourceTap$MemorySourceScheme
  • elephantdb.cascading.ElephantScheme
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.