Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.InputSplit

167
168
169
170
171
172
173
174
175
176
      } catch (HiveException e) {
        throw new RuntimeException(
            "Unable to get metadata for input table split" + split.getPath());
      }
    }
    InputSplit retA[] = newSplits.toArray((new FileSplit[newSplits.size()]));
    l4j.info("Number of input splits: " + splits.length + " new input splits: "
        + retA.length + ", sum of split lengths: " + sumSplitLengths);
    return retA;
  }
View Full Code Here
206
207
208
209
210
211
212
213
214
215
216
  public RecordReader getRecordReader(InputSplit split, JobConf job,
      Reporter reporter) throws IOException {

    HiveInputSplit hsplit = (HiveInputSplit) split;
   
    InputSplit inputSplit = hsplit.getInputSplit();
    String inputFormatClassName = null;
    Class inputFormatClass = null;
    try {
      inputFormatClassName = hsplit.inputFormatClassName();
      inputFormatClass = job.getClassByName(inputFormatClassName);
View Full Code Here
32
33
34
35
36
37
38
39
40
  public final ExpectedSuppliedException thrown = ExpectedSuppliedException
      .none();

  @Test
  public void testGetInputSplitForMapper() {
    final InputSplit split = new MockReporter(MockReporter.ReporterType.Mapper,
        null, new Path("/")).getInputSplit();
    assertTrue(null != split);
  }
View Full Code Here
319
320
321
322
323
324
325
326
327
328
329
330
331
    InputSplit[] splits = aif.getSplits(job, 1);

    Assert.assertEquals(1, splits.length);

    InputSplit split = splits[0];

    Assert.assertEquals(RangeInputSplit.class, split.getClass());

    RangeInputSplit risplit = (RangeInputSplit) split;

    Assert.assertEquals(username, risplit.getPrincipal());
    Assert.assertEquals(table, risplit.getTableName());
View Full Code Here
275
276
277
278
279
280
281
282
283
284
285
    return serialized;
  }

  private FileSplit getFileSplitFromEvent(RootInputDataInformationEvent event)
      throws IOException {
    InputSplit inputSplit = null;
    if (event.getDeserializedUserPayload() != null) {
      inputSplit = (InputSplit) event.getDeserializedUserPayload();
    } else {
      MRSplitProto splitProto = MRSplitProto.parseFrom(event.getUserPayload());
      SerializationFactory serializationFactory = new SerializationFactory(
View Full Code Here
191
192
193
194
195
196
197
198
199
200
201
  public RecordReader getRecordReader(InputSplit split, JobConf job,
      Reporter reporter) throws IOException {

    HiveInputSplit hsplit = (HiveInputSplit)split;

    InputSplit inputSplit = hsplit.getInputSplit();
    String inputFormatClassName = null;
    Class inputFormatClass = null;
    try {
      inputFormatClassName = hsplit.inputFormatClassName();
      inputFormatClass = job.getClassByName(inputFormatClassName);
View Full Code Here
189
190
191
192
193
194
195
196
197
198
199
  public RecordReader getRecordReader(InputSplit split, JobConf job,
      Reporter reporter) throws IOException {

    HiveInputSplit hsplit = (HiveInputSplit)split;

    InputSplit inputSplit = hsplit.getInputSplit();
    String inputFormatClassName = null;
    Class inputFormatClass = null;
    try {
      inputFormatClassName = hsplit.inputFormatClassName();
      inputFormatClass = job.getClassByName(inputFormatClassName);
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
  }

  @Override
  public RecordReader<LongWritable, Text> getRecordReader(
      InputSplit split, JobConf job, Reporter reporter) throws IOException {
    InputSplit targetSplit = ((SymlinkTextInputSplit)split).getTargetSplit();

    // The target data is in TextInputFormat.
    TextInputFormat inputFormat = new TextInputFormat();
    inputFormat.configure(job);
    RecordReader innerReader = null;
View Full Code Here
204
205
206
207
208
209
210
211
212
213
214
  public RecordReader getRecordReader(InputSplit split, JobConf job,
      Reporter reporter) throws IOException {

    HiveInputSplit hsplit = (HiveInputSplit) split;

    InputSplit inputSplit = hsplit.getInputSplit();
    String inputFormatClassName = null;
    Class inputFormatClass = null;
    try {
      inputFormatClassName = hsplit.inputFormatClassName();
      inputFormatClass = job.getClassByName(inputFormatClassName);
View Full Code Here
186
187
188
189
190
191
192
193
    outMeta.close();
  }

  public static void generateInputSplit(FileSystem fs, Path workDir, JobConf jobConf, Path mapInput) throws IOException {
    jobConf.setInputFormat(SequenceFileInputFormat.class);
    InputSplit split = createInputSplit(fs, workDir, jobConf, mapInput);
    writeSplitFiles(fs, jobConf, split);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.InputSplit

  • com.ebay.erl.mobius.core.mapred.MobiusInputSampler
  • com.sap.hadoop.windowing.io.TableWindowingInput
  • com.taobao.zeus.jobs.sub.tool.DataPreviewJob
  • org.apache.accumulo.core.client.mapred.AccumuloInputFormatTest
  • org.apache.drill.exec.store.hive.HiveSubScan
  • org.apache.hadoop.contrib.mongoreduce.MongoStreamInputFormat
  • org.apache.hadoop.hbase.mapred.TestTableSnapshotInputFormat
  • org.apache.hadoop.hive.accumulo.mr.TestHiveAccumuloTableInputFormat
  • org.apache.hadoop.hive.ql.exec.StatsNoJobTask
  • org.apache.hadoop.hive.ql.exec.StatsNoJobTask$StatsCollection
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.