Examples of InputSplit

eu.stratosphere.core.io.InputSplit
This interface must be implemented by all kind of input splits that can be assigned to input formats.
org.apache.flink.core.io.InputSplit
This interface must be implemented by all kind of input splits that can be assigned to input formats.
org.apache.hadoop.mapred.InputSplit
The description of the data for a single map task. @author Owen O'Malley
org.apache.hadoop.mapreduce.InputSplit
InputSplit represents the data to be processed by an individual {@link Mapper}.
Typically, it presents a byte-oriented view on the input and is the responsibility of {@link RecordReader} of the job to process this and presenta record-oriented view. @see InputFormat @see RecordReader
org.apache.hama.bsp.InputSplit

Examples of org.apache.hadoop.mapreduce.InputSplit

    HowlSchema schema = buildHiveSchema();
    RCFileInputDriver sd = new RCFileInputDriver();
    JobContext jc = new JobContext(conf, new JobID());
    sd.setInputPath(jc, file.toString());
    InputFormat<?,?> iF = sd.getInputFormat(null);
    InputSplit split = iF.getSplits(jc).get(0);
    sd.setOriginalSchema(jc, schema);
    sd.setOutputSchema(jc, schema);
    sd.initialize(jc, getProps());


    TaskAttemptContext tac = new TaskAttemptContext(conf, new TaskAttemptID());

View Full Code Here

Examples of org.apache.hadoop.mapreduce.InputSplit


    RCFileInputDriver sd = new RCFileInputDriver();
    JobContext jc = new JobContext(conf, new JobID());
    sd.setInputPath(jc, file.toString());
    InputFormat<?,?> iF = sd.getInputFormat(null);
    InputSplit split = iF.getSplits(jc).get(0);
    sd.setOriginalSchema(jc, buildHiveSchema());
    sd.setOutputSchema(jc, buildPrunedSchema());


    sd.initialize(jc, getProps());
    conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR,jc.getConfiguration().get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR));

View Full Code Here

Examples of org.apache.hadoop.mapreduce.InputSplit


    RCFileInputDriver sd = new RCFileInputDriver();
    JobContext jc = new JobContext(conf, new JobID());
    sd.setInputPath(jc, file.toString());
    InputFormat<?,?> iF = sd.getInputFormat(null);
    InputSplit split = iF.getSplits(jc).get(0);
    sd.setOriginalSchema(jc, buildHiveSchema());
    sd.setOutputSchema(jc, buildReorderedSchema());


    sd.initialize(jc, getProps());
    Map<String,String> map = new HashMap<String,String>(1);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.InputSplit

    public void readFields(DataInput input) throws IOException {
        String partitionInfoString = WritableUtils.readString(input);
        partitionInfo = (PartInfo) HowlUtil.deserialize(partitionInfoString);


        String baseSplitClassName = WritableUtils.readString(input);
        InputSplit split;
        try{
            Class<? extends InputSplit> splitClass =
                (Class<? extends InputSplit>) Class.forName(baseSplitClassName);


            //Class.forName().newInstance() does not work if the underlying

View Full Code Here

Examples of org.apache.hadoop.mapreduce.InputSplit

     * @see org.apache.hadoop.mapreduce.RecordReader#initialize(org.apache.hadoop.mapreduce.InputSplit, org.apache.hadoop.mapreduce.TaskAttemptContext)
     */
    @Override
    public void initialize(InputSplit split, TaskAttemptContext taskContext)
    throws IOException, InterruptedException {
        InputSplit baseSplit = split;


        if( split instanceof HowlSplit ) {
            baseSplit = ((HowlSplit) split).getBaseSplit();
        }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.InputSplit

      LOG.error("");


      LOG.error("Exception: ", e);
      LOG.error("On input: " + val);


      InputSplit is = context.getInputSplit();
      if (is instanceof FileSplit) {
        LOG.error("On input file: " + ((FileSplit)is).getPath());
      } else if (is instanceof CombineFileSplit) {
        LOG.error("On input file: "
          + context.getConfiguration().get("map.input.file"));

View Full Code Here

Examples of org.apache.hadoop.mapreduce.InputSplit

            return false;
        }
        if(reader != null){
            reader.close();
        }
        InputSplit curSplit = inpSplits.get(curSplitIndex);
        TaskAttemptContext tAContext = new TaskAttemptContext(conf, 
                new TaskAttemptID());
        reader = inputFormat.createRecordReader(curSplit, tAContext);
        reader.initialize(curSplit, tAContext);
        // create a dummy pigsplit - other than the actual split, the other

View Full Code Here

Examples of org.apache.hadoop.mapreduce.InputSplit

            return false;
        }
        if(reader != null){
            reader.close();
        }
        InputSplit curSplit = inpSplits.get(curSplitIndex);
        TaskAttemptContext tAContext = new TaskAttemptContext(conf, 
                new TaskAttemptID());
        reader = inputFormat.createRecordReader(curSplit, tAContext);
        reader.initialize(curSplit, tAContext);
        // create a dummy pigsplit - other than the actual split, the other

View Full Code Here

Examples of org.apache.hadoop.mapreduce.InputSplit


    List<InputSplit> splits = aif.getSplits(job);


    Assert.assertEquals(1, splits.size());


    InputSplit split = splits.get(0);


    Assert.assertEquals(RangeInputSplit.class, split.getClass());


    RangeInputSplit risplit = (RangeInputSplit) split;


    Assert.assertEquals(username, risplit.getPrincipal());
    Assert.assertEquals(table, risplit.getTable());

View Full Code Here

Examples of org.apache.hadoop.mapreduce.InputSplit


    HBaseTestingUtility.SeenRowTracker rowTracker = new HBaseTestingUtility.SeenRowTracker(startRow, stopRow);


    for (int i = 0; i < splits.size(); i++) {
      // validate input split
      InputSplit split = splits.get(i);
      Assert.assertTrue(split instanceof TableSnapshotRegionSplit);


      // validate record reader
      TaskAttemptContext taskAttemptContext = mock(TaskAttemptContext.class);
      when(taskAttemptContext.getConfiguration()).thenReturn(job.getConfiguration());

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.