Examples of org.apache.hadoop.mapred.InputSplit

org.apache.hadoop.mapred.InputSplit
The description of the data for a single map task. @author Owen O'Malley

          boolean statsAvailable = false;
          for(FileStatus file: fileList) {
            if (!file.isDir()) {
              InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtils.newInstance(
                  table.getInputFormatClass(), jc);
              InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0, new String[] { table
                  .getDataLocation().toString() });
              org.apache.hadoop.mapred.RecordReader<?, ?> recordReader = (org.apache.hadoop.mapred.RecordReader<?, ?>) inputFormat
                  .getRecordReader(dummySplit, jc, Reporter.NULL);
              StatsProvidingRecordReader statsRR;
              if (recordReader instanceof StatsProvidingRecordReader) {

View Full Code Here

        boolean statsAvailable = false;
        for(FileStatus file: fileList) {
          if (!file.isDir()) {
            InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtils.newInstance(
                partn.getInputFormatClass(), jc);
            InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0,
                new String[] { partn.getLocation() });
            Object recordReader = inputFormat.getRecordReader(dummySplit, jc, Reporter.NULL);
            StatsProvidingRecordReader statsRR;
            if (recordReader instanceof StatsProvidingRecordReader) {
              statsRR = (StatsProvidingRecordReader) recordReader;

View Full Code Here

    return serialized;
  }


  private FileSplit getFileSplitFromEvent(RootInputDataInformationEvent event)
      throws IOException {
    InputSplit inputSplit = null;
    if (event.getDeserializedUserPayload() != null) {
      inputSplit = (InputSplit) event.getDeserializedUserPayload();
    } else {
      MRSplitProto splitProto = MRSplitProto.parseFrom(event.getUserPayload());
      SerializationFactory serializationFactory = new SerializationFactory(

View Full Code Here

  }


  @Override
  public RecordReader<LongWritable, Text> getRecordReader(
      InputSplit split, JobConf job, Reporter reporter) throws IOException {
    InputSplit targetSplit = ((SymlinkTextInputSplit)split).getTargetSplit();


    // The target data is in TextInputFormat.
    TextInputFormat inputFormat = new TextInputFormat();
    inputFormat.configure(job);
    RecordReader innerReader = null;

View Full Code Here

  public RecordReader getRecordReader(InputSplit split, JobConf job,
      Reporter reporter) throws IOException {


    HiveInputSplit hsplit = (HiveInputSplit) split;


    InputSplit inputSplit = hsplit.getInputSplit();
    String inputFormatClassName = null;
    Class inputFormatClass = null;
    try {
      inputFormatClassName = hsplit.inputFormatClassName();
      inputFormatClass = job.getClassByName(inputFormatClassName);

View Full Code Here

      } catch (HiveException e) {
        throw new RuntimeException(
            "Unable to get metadata for input table split" + split.getPath());
      }
    }
    InputSplit retA[] = newSplits.toArray((new FileSplit[newSplits.size()]));
    l4j.info("Number of input splits: " + splits.length + " new input splits: "
        + retA.length);
    return retA;
  }

View Full Code Here

  public RecordReader getRecordReader(InputSplit split, JobConf job,
      Reporter reporter) throws IOException {


    HiveInputSplit hsplit = (HiveInputSplit) split;


    InputSplit inputSplit = hsplit.getInputSplit();
    String inputFormatClassName = null;
    Class inputFormatClass = null;
    try {
      inputFormatClassName = hsplit.inputFormatClassName();
      inputFormatClass = job.getClassByName(inputFormatClassName);

View Full Code Here

  }


  @Override
  public RecordReader<LongWritable, Text> getRecordReader(
      InputSplit split, JobConf job, Reporter reporter) throws IOException {
    InputSplit targetSplit = ((SymlinkTextInputSplit)split).getTargetSplit();


    // The target data is in TextInputFormat.
    TextInputFormat inputFormat = new TextInputFormat();
    inputFormat.configure(job);
    return inputFormat.getRecordReader(targetSplit, job, reporter);

View Full Code Here

      String[] hosts = MongoInputFormat.hostsForShard((String) shard.get("host"), primaryOk);
      
      for(String h : hosts)
        System.out.println("host:" + h);
          
      InputSplit split = new MongoStreamInputSplit(hosts);
      splits.add(split);
    }
    
    ret = new InputSplit[splits.size()];
    return splits.toArray(ret);

View Full Code Here

    HBaseTestingUtility.SeenRowTracker rowTracker =
      new HBaseTestingUtility.SeenRowTracker(startRow, stopRow);


    for (int i = 0; i < splits.length; i++) {
      // validate input split
      InputSplit split = splits[i];
      Assert.assertTrue(split instanceof TableSnapshotInputFormat.TableSnapshotRegionSplit);


      // validate record reader
      OutputCollector collector = mock(OutputCollector.class);
      Reporter reporter = mock(Reporter.class);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapred.InputSplit

com.ebay.erl.mobius.core.mapred.MobiusInputSampler

com.sap.hadoop.windowing.io.TableWindowingInput

com.taobao.zeus.jobs.sub.tool.DataPreviewJob

org.apache.accumulo.core.client.mapred.AccumuloInputFormatTest

org.apache.drill.exec.store.hive.HiveSubScan

org.apache.hadoop.contrib.mongoreduce.MongoStreamInputFormat

org.apache.hadoop.hbase.mapred.TestTableSnapshotInputFormat

org.apache.hadoop.hive.accumulo.mr.TestHiveAccumuloTableInputFormat

org.apache.hadoop.hive.ql.exec.StatsNoJobTask

org.apache.hadoop.hive.ql.exec.StatsNoJobTask$StatsCollection

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.