Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.InputSplit

214
215
216
217
218
219
220
221
222
223
224
  public RecordReader getRecordReader(InputSplit split, JobConf job,
      Reporter reporter) throws IOException {

    HiveInputSplit hsplit = (HiveInputSplit) split;

    InputSplit inputSplit = hsplit.getInputSplit();
    String inputFormatClassName = null;
    Class inputFormatClass = null;
    try {
      inputFormatClassName = hsplit.inputFormatClassName();
      inputFormatClass = job.getClassByName(inputFormatClassName);
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
  }

  @Override
  public RecordReader<LongWritable, Text> getRecordReader(
      InputSplit split, JobConf job, Reporter reporter) throws IOException {
    InputSplit targetSplit = ((SymlinkTextInputSplit)split).getTargetSplit();

    // The target data is in TextInputFormat.
    TextInputFormat inputFormat = new TextInputFormat();
    inputFormat.configure(job);
    RecordReader innerReader = null;
View Full Code Here
401
402
403
404
405
406
407
408
409
410
411
  public void testIteratorNotInSplitsCompensation() throws Exception {
    FileInputFormat.addInputPath(conf, new Path("unused"));
    InputSplit[] splits = inputformat.getSplits(conf, 0);

    assertEquals(1, splits.length);
    InputSplit split = splits[0];

    IteratorSetting is = new IteratorSetting(1, PrimitiveComparisonFilter.FILTER_PREFIX + 1,
        PrimitiveComparisonFilter.class);

    is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
View Full Code Here
170
171
172
173
174
175
176
177
178
179
180
    MapWork work = populateMapWork(jobConf, inputName);
    Multimap<Integer, InputSplit> bucketSplitMultiMap =
        ArrayListMultimap.<Integer, InputSplit> create();

    int i = 0;
    InputSplit prevSplit = null;
    for (InputSplit s : splits) {
      // this is the bit where we make sure we don't group across partition
      // schema boundaries
      if (schemaEvolved(s, prevSplit, groupAcrossFiles, work)) {
        ++i;
View Full Code Here
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
    byte[] serialized = dob.getData();
    return UserPayload.create(ByteBuffer.wrap(serialized));
  }

  private FileSplit getFileSplitFromEvent(InputDataInformationEvent event) throws IOException {
    InputSplit inputSplit = null;
    if (event.getDeserializedUserPayload() != null) {
      inputSplit = (InputSplit) event.getDeserializedUserPayload();
    } else {
      MRSplitProto splitProto = MRSplitProto.parseFrom(ByteString.copyFrom(event.getUserPayload()));
      SerializationFactory serializationFactory = new SerializationFactory(new Configuration());
      inputSplit = MRInputHelpers.createOldFormatSplitFromUserPayload(splitProto, serializationFactory);
    }

    if (!(inputSplit instanceof FileSplit)) {
      throw new UnsupportedOperationException(
          "Cannot handle splits other than FileSplit for the moment. Current input split type: "
              + inputSplit.getClass().getSimpleName());
    }
    return (FileSplit) inputSplit;
  }
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
      // 忽略目录
      if (f.isDir()) {
        continue;
      }

      @SuppressWarnings("deprecation")
      InputSplit split = new FileSplit(f.getPath(), 0, f.getLen(),
          new JobConf(conf));
      reader = inputFormat.getRecordReader(split, confQ, Reporter.NULL);
      Writable key = null;
      Text textValue = new Text();
View Full Code Here
242
243
244
245
246
247
248
249
250
251
252
          boolean statsAvailable = false;
          for(FileStatus file: fileList) {
            if (!file.isDir()) {
              InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtils.newInstance(
                  table.getInputFormatClass(), jc);
              InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0, new String[] { table
                  .getDataLocation().toString() });
              org.apache.hadoop.mapred.RecordReader<?, ?> recordReader = (org.apache.hadoop.mapred.RecordReader<?, ?>) inputFormat
                  .getRecordReader(dummySplit, jc, Reporter.NULL);
              StatsProvidingRecordReader statsRR;
              if (recordReader instanceof StatsProvidingRecordReader) {
View Full Code Here
149
150
151
152
153
154
155
156
157
158
159
        boolean statsAvailable = false;
        for(FileStatus file: fileList) {
          if (!file.isDir()) {
            InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtils.newInstance(
                partn.getInputFormatClass(), jc);
            InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0,
                new String[] { partn.getLocation() });
            org.apache.hadoop.mapred.RecordReader<?, ?> recordReader =
                (org.apache.hadoop.mapred.RecordReader<?, ?>)
                inputFormat.getRecordReader(dummySplit, jc, Reporter.NULL);
            StatsProvidingRecordReader statsRR;
View Full Code Here
25
26
27
28
29
30
31
32
@SuppressWarnings("deprecation")
public class TestMockReporter {

  @Test
  public void testGetInputSplitForMapper() {
    InputSplit split = new MockReporter(MockReporter.ReporterType.Mapper, null).getInputSplit();
    assertTrue(null != split);
  }
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
  public HiveReadEntry getHiveReadEntry() {
    return hiveReadEntry;
  }

  public static InputSplit deserializeInputSplit(String base64, String className) throws IOException, ReflectiveOperationException{
    InputSplit split;
    if (Class.forName(className) == FileSplit.class) {
      split = new FileSplit((Path) null, 0, 0, (String[])null);
    } else {
      split = (InputSplit) Class.forName(className).getConstructor().newInstance();
    }
    ByteArrayDataInput byteArrayDataInput = ByteStreams.newDataInput(Base64.decodeBase64(base64));
    split.readFields(byteArrayDataInput);
    return split;
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.InputSplit

  • com.ebay.erl.mobius.core.mapred.MobiusInputSampler
  • com.sap.hadoop.windowing.io.TableWindowingInput
  • com.taobao.zeus.jobs.sub.tool.DataPreviewJob
  • org.apache.accumulo.core.client.mapred.AccumuloInputFormatTest
  • org.apache.drill.exec.store.hive.HiveSubScan
  • org.apache.hadoop.contrib.mongoreduce.MongoStreamInputFormat
  • org.apache.hadoop.hbase.mapred.TestTableSnapshotInputFormat
  • org.apache.hadoop.hive.accumulo.mr.TestHiveAccumuloTableInputFormat
  • org.apache.hadoop.hive.ql.exec.StatsNoJobTask
  • org.apache.hadoop.hive.ql.exec.StatsNoJobTask$StatsCollection
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.