Examples of org.apache.hadoop.mapred.lib.CombineFileSplit

org.apache.hadoop.mapred.lib.CombineFileSplit
A sub-collection of input files. Unlike {@link org.apache.hadoop.mapred.FileSplit}, CombineFileSplit * class does not represent a split of a file, but a split of input files into smaller sets. A split may contain blocks from different file but all the blocks in the same split are probably local to some rack
CombineFileSplit can be used to implement {@link org.apache.hadoop.mapred.RecordReader}'s, with reading one record per file. @see org.apache.hadoop.mapred.FileSplit @see CombineFileInputFormat


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here

    @Override
    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here


      InputSplit[] splits = super.getSplits(job, numSplits);


      ArrayList<InputSplitShim> inputSplitShims = new ArrayList<InputSplitShim>();
      for (int pos = 0; pos < splits.length; pos++) {
        CombineFileSplit split = (CombineFileSplit) splits[pos];
        Set<Integer> dirIndices = getDirIndices(split.getPaths(), job);
        if (dirIndices.size() != split.getPaths().length) {
          List<Path> prunedPaths = prune(dirIndices, Arrays.asList(split.getPaths()));
          List<Long> prunedStartOffsets = prune(dirIndices, Arrays.asList(
            ArrayUtils.toObject(split.getStartOffsets())));
          List<Long> prunedLengths = prune(dirIndices, Arrays.asList(
            ArrayUtils.toObject(split.getLengths())));
          inputSplitShims.add(new InputSplitShim(job, prunedPaths.toArray(new Path[prunedPaths.size()]),
            Longs.toArray(prunedStartOffsets),
            Longs.toArray(prunedLengths), split.getLocations()));
        }
      }
      return inputSplitShims.toArray(new InputSplitShim[inputSplitShims.size()]);
    }

View Full Code Here

    @Override
    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.mapred.lib.CombineFileSplit

com.alexholmes.hadooputils.combine.common.mapred.SplitMetricsCombineInputFormat

org.apache.hadoop.fs.Path

org.apache.hadoop.hive.shims.Hadoop20Shims$CombineFileInputFormatShim

org.apache.hadoop.hive.shims.Hadoop20SShims$CombineFileInputFormatShim

org.apache.hadoop.hive.shims.Hadoop23Shims$CombineFileInputFormatShim

org.apache.hadoop.hive.shims.HadoopShimsSecure$CombineFileInputFormatShim

org.terrier.structures.indexing.singlepass.hadoop.MultiFileCollectionInputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.