Examples of FsInput

Examples of org.apache.avro.mapred.FsInput

73
74
75
76
77
78
79
80
81
82
83
  @Override
  public Iterator<T> read(FileSystem fs, final Path path) {
    this.mapFn.initialize();
    try {
      FsInput fsi = new FsInput(path, fs.getConf());
      final DataFileReader<T> reader = new DataFileReader<T>(fsi, recordReader);
      return new AutoClosingIterator<T>(reader, new UnmodifiableIterator<T>() {
        @Override
        public boolean hasNext() {
          return reader.hasNext();
View Full Code Here

Examples of org.apache.avro.mapred.FsInput

115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
  /**
   * Read the Avro schema from the first file in the input directory.
   */
  private Schema readSchema(Path inputDir, Configuration conf) throws IOException {
    FsInput fsInput = null;
    FileReader<Object> reader = null;
    try {
      fsInput = new FsInput(new Path(inputDir, "part-m-00000.avro"), conf);
      reader = DataFileReader.openReader(fsInput, new GenericDatumReader<Object>());
      return reader.getSchema();
    } finally {
      IOUtils.closeStream(fsInput);
      IOUtils.closeStream(reader);
View Full Code Here

Examples of org.apache.avro.mapred.FsInput

391
392
393
394
395
396
397
398
399
400
401
        if (fstat == null || fstat.length == 0) {
          throw new IllegalArgumentException("No valid files found in directory: " + path);
        }
        path = fstat[0].getPath();
      }
      reader = new DataFileReader(new FsInput(path, conf), new GenericDatumReader<GenericRecord>());
      return reader.getSchema();
    } catch (IOException e) {
      throw new RuntimeException("Error reading schema from path: "  + path, e);
    } finally {
      if (reader != null) {
View Full Code Here

Examples of org.apache.avro.mapred.FsInput

49
50
51
52
53
54
55
56
57
58
59
  @Override
  public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException, InterruptedException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration conf = context.getConfiguration();
    SeekableInput in = new FsInput(split.getPath(), conf);
    DatumReader<T> datumReader = AvroMode
        .fromConfiguration(context.getConfiguration())
        .getReader(schema);
    this.reader = DataFileReader.openReader(in, datumReader);
    reader.sync(split.getStart()); // sync to start
View Full Code Here

Examples of org.apache.avro.mapred.FsInput

69
70
71
72
73
74
75
76
77
78
79
  public Iterator<T> read(FileSystem fs, final Path path) {
    AvroMode mode = AvroMode.fromType(atype).withFactoryFromConfiguration(fs.getConf());
    final DatumReader recordReader = reader == null ? mode.getReader(atype.getSchema()) : reader;
    this.mapFn.initialize();
    try {
      FsInput fsi = new FsInput(path, fs.getConf());
      final DataFileReader<T> reader = new DataFileReader<T>(fsi, recordReader);
      return new AutoClosingIterator<T>(reader, new UnmodifiableIterator<T>() {
        @Override
        public boolean hasNext() {
          return reader.hasNext();
View Full Code Here

Examples of org.apache.avro.mapred.FsInput

100
101
102
103
104
105
106
107
108
    start  = fsplit.getStart();
    end    = fsplit.getStart() + fsplit.getLength();
    DatumReader<GenericData.Record> datumReader
      = new GenericDatumReader<GenericData.Record>(schema);
    reader = DataFileReader.openReader(
        new FsInput(fsplit.getPath(), tc.getConfiguration()),
        datumReader);
    reader.sync(start);
  }
View Full Code Here

Examples of org.apache.avro.mapred.FsInput

109
110
111
112
113
114
115
116
117
    start  = fsplit.getStart();
    end    = fsplit.getStart() + fsplit.getLength();
    DatumReader<GenericData.Array<Object>> datumReader
      = new GenericDatumReader<GenericData.Array<Object>>(schema);
    reader = DataFileReader.openReader(
        new FsInput(fsplit.getPath(), tc.getConfiguration()),
        datumReader);
    reader.sync(start);
  }
View Full Code Here

Examples of org.apache.gora.avro.mapreduce.FsInput

86
87
88
89
90
91
92
93
94
95
96
  }
  @Override
  protected Result<K,T> executePartial(FileSplitPartitionQuery<K,T> query) {
    try{
      FsInput fsInput = createFsInput();
      DataFileReader<T> reader = createReader(fsInput);
      return new DataFileAvroResult<K, T>(this, query, reader, fsInput
          , query.getStart(), query.getLength());
    } catch(IOException ex){
      LOG.error(ex.getMessage());
View Full Code Here

Examples of org.apache.gora.avro.mapreduce.FsInput

103
104
105
106
107
108
109
    return new DataFileReader<T>(fsInput, getDatumReader());
  }
 
  private FsInput createFsInput() throws IOException {
    Path path = new Path(getInputPath());
    return new FsInput(path, getConf());
  }
View Full Code Here

Examples of org.apache.gora.avro.mapreduce.FsInput

69
70
71
72
73
74
75
76
77
78
  }
  @Override
  protected Result<K,T> executePartial(FileSplitPartitionQuery<K,T> query)
    throws IOException {
    FsInput fsInput = createFsInput();
    DataFileReader<T> reader = createReader(fsInput);
    return new DataFileAvroResult<K, T>(this, query, reader, fsInput
        , query.getStart(), query.getLength());
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.