Package org.apache.avro.mapred

Examples of org.apache.avro.mapred.FsInput

51
52
53
54
55
56
57
58
59
60
61
  @Override
  public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException, InterruptedException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration conf = context.getConfiguration();
    SeekableInput in = new FsInput(split.getPath(), conf);
    DatumReader<T> datumReader = null;
    if (context.getConfiguration().getBoolean(AvroJob.INPUT_IS_REFLECT, true)) {
      ReflectDataFactory factory = Avros.getReflectDataFactory(conf);
      datumReader = factory.getReader(schema);
    } else {
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
    if(latest != null) {
      gdr.setExpected(latest);
    }

    this.reader = new DataFileReader<GenericRecord>(new FsInput(split.getPath(), job), gdr);
    this.reader.sync(split.getStart());
    this.start = reader.tell();
    this.stop = split.getStart() + split.getLength();
    this.recordReaderID = new UID();
  }
View Full Code Here
222
223
224
225
226
227
228
229
      Schema recordSchema = AvroKeyValue.getSchema(mKeySchema, options.getValueSchema());
      DatumReader<GenericRecord> datumReader =
        model.createDatumReader(recordSchema);
      mDataFileReader =
        new DataFileReader<GenericRecord>
        (new FsInput(dataFilePath, options.getConfiguration()), datumReader);
     
    }
View Full Code Here
304
305
306
307
308
309
310
311
312
313
314
    private <K> NavigableMap<K, Long> loadIndexFile(
        Configuration conf, Path path, Schema keySchema) throws IOException {
      DatumReader<GenericRecord> datumReader = model.createDatumReader(
          AvroKeyValue.getSchema(keySchema, Schema.create(Schema.Type.LONG)));
      DataFileReader<GenericRecord> fileReader = new DataFileReader<GenericRecord>(
          new FsInput(path, conf), datumReader);

      NavigableMap<K, Long> index;
      if (Schema.create(Schema.Type.STRING).equals(keySchema)) {
        // Because Avro STRING types are mapped to the Java CharSequence class that does not
        // mandate the implementation of Comparable, we need to specify a special
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
    if (nargs.size() != 1) {
      err.println("Expected 1 arg: input_file");
      p.printHelpOn(err);
      return 1;
    }
    FsInput in = Util.openSeekableFromFS(args.get(0));
    DataFileReader<Void> reader =
      new DataFileReader<Void>(in, new GenericDatumReader<Void>());
    if (keyName != null) {
      byte[] value = reader.getMeta(keyName);
      if (value != null) {
View Full Code Here
105
106
107
108
109
110
111
   * @param filename The filename to be opened
   * @throws IOException
   */
  static FsInput openSeekableFromFS(String filename)
      throws IOException {      
    return new FsInput(new Path(filename), new Configuration());
  }
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
    if(latest != null) {
      gdr.setExpected(latest);
    }

    this.reader = new DataFileReader<GenericRecord>(new FsInput(split.getPath(), job), gdr);
    this.reader.sync(split.getStart());
    this.start = reader.tell();
    this.stop = split.getStart() + split.getLength();
    this.recordReaderID = new UID();
  }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
    if (nargs.size() != 1) {
      err.println("Expected 1 arg: input_file");
      p.printHelpOn(err);
      return 1;
    }
    FsInput in = Util.openSeekableFromFS(args.get(0));
    DataFileReader<Void> reader =
      new DataFileReader<Void>(in, new GenericDatumReader<Void>());
    if (keyName != null) {
      byte[] value = reader.getMeta(keyName);
      if (value != null) {
View Full Code Here
67
68
69
70
71
72
73
74
75
76
    GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>();

    if(latest != null) gdr.setExpected(latest);

    this.reader = new DataFileReader<GenericRecord>(new FsInput(split.getPath(), job), gdr);
    this.reader.sync(split.getStart());
    this.start = reader.tell();
    this.stop = split.getStart() + split.getLength();
  }
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
  }
 
  public void initialize(FileSplit split,Configuration conf)
      throws IOException, InterruptedException{
      init(DataFileReader.openReader
        (new FsInput(split.getPath(), conf),
         conf.getBoolean(AvroJob.INPUT_IS_REFLECT, false)
         ? new ReflectDatumReader<T>(AvroJob.getInputSchema(conf))
         : new SpecificDatumReader<T>(AvroJob.getInputSchema(conf))),
        split);
  }
View Full Code Here
TOP

Related Classes of org.apache.avro.mapred.FsInput

  • com.cloudera.sqoop.TestAvroImport
  • com.cloudera.sqoop.testutil.LobAvroImportTestCase
  • com.datasalt.pangool.tuplemr.avro.AvroRecordReader
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleInputFormat$TupleInputReader
  • com.linkedin.camus.sweeper.mapreduce.CamusSweeperAvroKeyJob
  • com.linkedin.haivvreo.AvroGenericRecordReader
  • com.linkedin.pig.FastAvroRecordReader
  • MaxWidgetIdGenericAvro
  • org.apache.avro.hadoop.file.SortedKeyValueFile$Reader
  • org.apache.avro.hadoop.file.TestSortedKeyValueFile
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.