Examples of org.apache.hadoop.mapred.lib.CombineFileRecordReader

org.apache.hadoop.mapred.lib.CombineFileRecordReader
A generic RecordReader that can hand out different recordReaders for each chunk in a {@link CombineFileSplit}. A CombineFileSplit can combine data chunks from multiple files. This class allows using different RecordReaders for processing these data chunks from different files. @see CombineFileSplit

              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    FileInputFormat.setInputPaths(job, workDir);
    CombineFileInputFormat format = new CombineFileInputFormat() {
      @Override
      public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException {
        return new CombineFileRecordReader(job, (CombineFileSplit)split, reporter, CombineFileRecordReader.class);
      }
    };
    final int SIZE_SPLITS = 1;
    LOG.info("Trying to getSplits with splits = " + SIZE_SPLITS);
    InputSplit[] splits = format.getSplits(job, SIZE_SPLITS);

View Full Code Here

              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    FileInputFormat.setInputPaths(job, workDir);
    CombineFileInputFormat format = new CombineFileInputFormat() {
      @Override
      public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException {
        return new CombineFileRecordReader(job, (CombineFileSplit)split, reporter, CombineFileRecordReader.class);
      }
    };
    final int SIZE_SPLITS = 1;
    LOG.info("Trying to getSplits with splits = " + SIZE_SPLITS);
    InputSplit[] splits = format.getSplits(job, SIZE_SPLITS);

View Full Code Here

              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    FileInputFormat.setInputPaths(job, workDir);
    CombineFileInputFormat format = new CombineFileInputFormat() {
      @Override
      public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException {
        return new CombineFileRecordReader(job, (CombineFileSplit)split, reporter, CombineFileRecordReader.class);
      }
    };
    final int SIZE_SPLITS = 1;
    LOG.info("Trying to getSplits with splits = " + SIZE_SPLITS);
    InputSplit[] splits = format.getSplits(job, SIZE_SPLITS);

View Full Code Here

    {
    private Configuration conf;


    public RecordReader getRecordReader( InputSplit split, JobConf job, Reporter reporter ) throws IOException
      {
      return new CombineFileRecordReader( job, (CombineFileSplit) split, reporter, CombineFileRecordReaderWrapper.class );
      }

View Full Code Here


public class CombinePcapInputFormat extends CombineFileInputFormat<LongWritable, ObjectWritable> {
  @SuppressWarnings({ "unchecked", "rawtypes" })
  @Override
  public RecordReader<LongWritable, ObjectWritable> getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException {
    return new CombineFileRecordReader(job, (CombineFileSplit)split, reporter, CombinePcapRecordReader.class);
  }

View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapred.lib.CombineFileRecordReader

cascading.tap.hadoop.Hfs$CombinedInputFormat

net.ripe.hadoop.pcap.io.CombinePcapInputFormat

org.apache.hadoop.mapred.TestCombineFileInputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.