Examples of TextInputFormat

Examples of org.apache.hadoop.mapreduce.lib.input.TextInputFormat

94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
    FileInputFormat.setInputPaths(job, dataPath);

    setMaxSplitSize(job.getConfiguration(), dataPath, numMaps);

    // retrieve the splits
    TextInputFormat input = new TextInputFormat();
    List<InputSplit> splits = input.getSplits(job);
    assertEquals(numMaps, splits.size());

    InputSplit[] sorted = new InputSplit[numMaps];
    splits.toArray(sorted);
    Builder.sortSplits(sorted);

    Step0Context context = new Step0Context(new Step0Mapper(), job.getConfiguration(),
                                            new TaskAttemptID(), numMaps);

    for (int p = 0; p < numMaps; p++) {
      InputSplit split = sorted[p];

      RecordReader<LongWritable, Text> reader = input.createRecordReader(split,
                                                                         context);
      reader.initialize(split, context);

      Step0Mapper mapper = new Step0Mapper();
      mapper.configure(p);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.TextInputFormat

169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
    FileInputFormat.setInputPaths(job, dataPath);

    setMaxSplitSize(job.getConfiguration(), dataPath, numMaps);

    // retrieve the splits
    TextInputFormat input = new TextInputFormat();
    List<InputSplit> splits = input.getSplits(job);
    assertEquals(numMaps, splits.size());

    InputSplit[] sorted = new InputSplit[numMaps];
    splits.toArray(sorted);
    Builder.sortSplits(sorted);

    List<Integer> keys = new ArrayList<Integer>();
    List<Step0Output> values = new ArrayList<Step0Output>();

    int[] expectedIds = new int[numMaps];

    TaskAttemptContext context = new TaskAttemptContext(job.getConfiguration(),
        new TaskAttemptID());

    for (int p = 0; p < numMaps; p++) {
      InputSplit split = sorted[p];
      RecordReader<LongWritable, Text> reader = input.createRecordReader(split,
          context);
      reader.initialize(split, context);

      Long firstKey = null;
      int size = 0;
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.TextInputFormat

320
321
322
323
324
325
326
327
328
        InputFormat result = null;
        if(inputAvroSchema != null) {
            result = new PigAvroInputFormat(
            inputAvroSchema, ignoreBadFiles, schemaToMergedSchemaMap, useMultipleSchemas);
        } else {
            result = new TextInputFormat();
        }
        return result;
    }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.TextInputFormat

38
39
40
41
42
43
44
    FileInputFormat.setInputPaths(job, location);
  }
 
  @Override
  public InputFormat getInputFormat() {
    return new TextInputFormat();
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.TextInputFormat

89
90
91
92
93
94
95
    @SuppressWarnings("unchecked")
    public InputFormat getInputFormat() throws IOException {
        // We will use TextInputFormat, the default Hadoop input format for
        // text.  It has a LongWritable key that we will ignore, and the value
        // is a Text (a string writable) that the JSON data is in.
        return new TextInputFormat();
    }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.TextInputFormat

93
94
95
96
97
98
99
    @SuppressWarnings("unchecked")
    public InputFormat getInputFormat() throws IOException {
        // We will use TextInputFormat, the default Hadoop input format for
        // text.  It has a LongWritable key that we will ignore, and the value
        // is a Text (a string writable) that the JSON data is in.
        return new TextInputFormat();
    }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.TextInputFormat

383
384
385
386
387
388
389
390
391
        InputFormat result = null;
        if(inputAvroSchema != null) {
            result = new PigAvroInputFormat(
            inputAvroSchema, ignoreBadFiles, schemaToMergedSchemaMap, useMultipleSchemas);
        } else {
            result = new TextInputFormat();
        }
        return result;
    }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.TextInputFormat

226
227
228
229
230
231
232
     * Methods called on the frontend
     */

    @Override
    public InputFormat getInputFormat() throws IOException {
        return new TextInputFormat();
    }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.TextInputFormat

72
73
74
75
76
77
78
  }
 
  @SuppressWarnings("unchecked")
  @Override
  public InputFormat getInputFormat() throws IOException {
      return new TextInputFormat();
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.TextInputFormat

59
60
61
62
63
64
65
        }
    }
   
    @Override
    public InputFormat getInputFormat() {
        return new TextInputFormat();
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.