Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.TextInputFormat$LineRecordReader$TextStuffer

51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
    // ---- set where we'll read the input files from -------------
    FileInputFormat.setInputPaths(job, input_path);

    // try splitting the file in a variety of sizes
    TextInputFormat format = new TextInputFormat();
    format.configure(job);

    int numSplits = 1;

    InputSplit[] splits = null;

    try {
      splits = format.getSplits(job, numSplits);
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
    // ---- set where we'll read the input files from -------------
    FileInputFormat.setInputPaths(job, input_path);

    // try splitting the file in a variety of sizes
    TextInputFormat format = new TextInputFormat();
    format.configure(job);

    int numSplits = 1;

    InputSplit[] splits = null;

    try {
      splits = format.getSplits(job, numSplits);
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
View Full Code Here
60
61
62
63
64
65
66
  private static class TextRecordReaderWrapper
    extends CombineFileRecordReaderWrapper<LongWritable,Text> {
    // this constructor signature is required by CombineFileRecordReader
    public TextRecordReaderWrapper(CombineFileSplit split, Configuration conf,
      Reporter reporter, Integer idx) throws IOException {
      super(new TextInputFormat(), split, conf, reporter, idx);
    }
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
      Collections.sort(columnIds);
      numCols = columnIds.size();
    }
    targetRecordCount = context.getConfig().getInt(ExecConstants.TEXT_LINE_READER_BATCH_SIZE);

    TextInputFormat inputFormat = new TextInputFormat();
    JobConf job = new JobConf();
    job.setInt("io.file.buffer.size", context.getConfig().getInt(ExecConstants.TEXT_LINE_READER_BUFFER_SIZE));
    job.setInputFormat(inputFormat.getClass());
    try {
      reader = inputFormat.getRecordReader(split, job, Reporter.NULL);
      key = reader.createKey();
      value = reader.createValue();
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
    int numSubTasks   = (args.length > 0 ? Integer.parseInt(args[0]) : 1);
    String dataInput = (args.length > 1 ? args[1] : "");
    String output    = (args.length > 2 ? args[2] : "");

    HadoopDataSource<LongWritable, Text> source = new HadoopDataSource<LongWritable, Text>(
        new TextInputFormat(), new JobConf(), "Input Lines");
    TextInputFormat.addInputPath(source.getJobConf(), new Path(dataInput));


    MapOperator mapper = MapOperator.builder(new TokenizeLine())
        .input(source)
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
    int numSubTasks   = (args.length > 0 ? Integer.parseInt(args[0]) : 1);
    String dataInput = (args.length > 1 ? args[1] : "");
    String output    = (args.length > 2 ? args[2] : "");
   
   
    HadoopDataSource source = new HadoopDataSource(new TextInputFormat(), new JobConf(), "Input Lines");
    TextInputFormat.addInputPath(source.getJobConf(), new Path(dataInput));
   
    // Example with Wrapper Converter
    HadoopDataSource<LongWritable,Text> sourceHadoopType = new HadoopDataSource<LongWritable, Text>(
        new TextInputFormat(), new JobConf(), "Input Lines", new WritableWrapperConverter<LongWritable, Text>());
    TextInputFormat.addInputPath(source.getJobConf(), new Path(dataInput));
   
    MapOperator mapper = MapOperator.builder(new TokenizeLine())
      .input(source)
      .name("Tokenize Lines")
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
    // Make TextInputFormat recognize .ssf via config
    JobConf job = new JobConf();
    job.set("fs.default.name", "file:///");
    // System.out.println("FS DEFAULT: " + job.get("fs.default.name"));
    job.set("io.compression.codecs", SimpleSeekableFormatCodec.class.getName());
    TextInputFormat textInputFormat = new TextInputFormat();
    textInputFormat.configure(job);

    // Open the file using TextInputFormat
    TextInputFormat.addInputPath(job, path);
    InputSplit[] splits = textInputFormat.getSplits(job, 1);
    Assert.assertEquals(1, splits.length);
    RecordReader<LongWritable, Text> recordReader = textInputFormat.getRecordReader(splits[0],
        job, Reporter.NULL);

    // Verify the data
    LongWritable key = recordReader.createKey();
    Text value = recordReader.createValue();
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.TextInputFormat$LineRecordReader$TextStuffer

  • com.cloudera.iterativereduce.io.TextRecordParser
  • com.cloudera.iterativereduce.irunit.IRUnitDriver
  • com.m6d.filecrush.crush.KeyValuePreservingTextInputFormat
  • eu.stratosphere.hadoopcompatibility.example.WordCount
  • eu.stratosphere.hadoopcompatibility.example.WordCountWithHadoopOutputFormat
  • org.apache.drill.exec.store.text.DrillTextRecordReader
  • org.apache.flink.hadoopcompatibility.mapred.example.HadoopMapredCompatWordCount
  • org.apache.flink.hadoopcompatibility.mapred.record.example.WordCount
  • org.apache.flink.hadoopcompatibility.mapred.record.example.WordCountWithOutputFormat
  • org.apache.hadoop.fs.FSDataInputStream
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.