Package org.apache.hadoop.mapreduce.lib.input

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

200
201
202
203
204
205
206
207
208
209
210
    conf.set(AggregatingRecordReader.RETURN_PARTIAL_MATCHES, Boolean.toString(false));
    File f = createFile(xml3);
   
    // Create FileSplit
    Path p = new Path(f.toURI().toString());
    WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
   
    // Initialize the RecordReader
    AggregatingRecordReader reader = new AggregatingRecordReader();
    reader.initialize(split, ctx);
    assertTrue(reader.nextKeyValue());
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
  public void testPartialXML2() throws Exception {
    File f = createFile(xml3);
   
    // Create FileSplit
    Path p = new Path(f.toURI().toString());
    WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
   
    // Initialize the RecordReader
    AggregatingRecordReader reader = new AggregatingRecordReader();
    reader.initialize(split, ctx);
    assertTrue(reader.nextKeyValue());
View Full Code Here
243
244
245
246
247
248
249
250
251
252
253
  public void testLineSplitting() throws Exception {
    File f = createFile(xml4);
   
    // Create FileSplit
    Path p = new Path(f.toURI().toString());
    WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
   
    // Initialize the RecordReader
    AggregatingRecordReader reader = new AggregatingRecordReader();
    reader.initialize(split, ctx);
    assertTrue(reader.nextKeyValue());
View Full Code Here
262
263
264
265
266
267
268
269
270
271
272
  @Test
  public void testNoEndTokenHandling() throws Exception {
    File f = createFile(xml5);
    // Create FileSplit
    Path p = new Path(f.toURI().toString());
    WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
   
    // Initialize the RecordReader
    AggregatingRecordReader reader = new AggregatingRecordReader();
    reader.initialize(split, ctx);
    assertTrue("Not enough records returned.", reader.nextKeyValue());
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
    Assert.assertNotNull(url);
    File data = new File(url.toURI());
    Path tmpFile = new Path(data.getAbsolutePath());
   
    // Setup the Mapper
    WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(tmpFile, 0, fs.pathToFile(tmpFile).length(), null),0);
    AggregatingRecordReader rr = new AggregatingRecordReader();
    Path ocPath = new Path(tmpFile, "oc");
    OutputCommitter oc = new FileOutputCommitter(ocPath, context);
    fs.deleteOnExit(ocPath);
    StandaloneStatusReporter sr = new StandaloneStatusReporter();
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
  public void testSerialization() throws IOException {
    Path testPath = new Path("/foo/bar");
    String[] hosts = new String[2];
    hosts[0] = "abcd";
    hosts[1] = "efgh";
    FileSplit fSplit = new FileSplit(testPath, 1, 2, hosts);
    WikipediaInputSplit split = new WikipediaInputSplit(fSplit, 7);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ObjectOutputStream out = new ObjectOutputStream(baos);
    split.write(out);
    out.close();
    baos.close();
   
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    DataInput in = new ObjectInputStream(bais);
   
    WikipediaInputSplit split2 = new WikipediaInputSplit();
    split2.readFields(in);
    Assert.assertTrue(bais.available() == 0);
    bais.close();
   
    Assert.assertTrue(split.getPartition() == split2.getPartition());
   
    FileSplit fSplit2 = split2.getFileSplit();
    Assert.assertTrue(fSplit.getPath().equals(fSplit2.getPath()));
    Assert.assertTrue(fSplit.getStart() == fSplit2.getStart());
    Assert.assertTrue(fSplit.getLength() == fSplit2.getLength());
   
    String[] hosts2 = fSplit2.getLocations();
    Assert.assertEquals(hosts.length, hosts2.length);
    for (int i = 0; i < hosts.length; i++) {
      Assert.assertEquals(hosts[i], hosts2[i]);
    }
  }
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
    Assert.assertNotNull(url);
    File data = new File(url.toURI());
    Path tmpFile = new Path(data.getAbsolutePath());
   
    // Setup the Mapper
    InputSplit split = new FileSplit(tmpFile, 0, fs.pathToFile(tmpFile).length(), null);
    AggregatingRecordReader rr = new AggregatingRecordReader();
    Path ocPath = new Path(tmpFile, "oc");
    OutputCommitter oc = new FileOutputCommitter(ocPath, context);
    fs.deleteOnExit(ocPath);
    StandaloneStatusReporter sr = new StandaloneStatusReporter();
View Full Code Here
239
240
241
242
243
244
245
246
247
248
249
250
    }

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
      if (first) {
        FileSplit split = (FileSplit) context.getInputSplit();
        Path path = split.getPath(); // current split path
        lvalue.set(path.getName());
        context.write(key, lvalue);

        first = false;
      }
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
    @Override
    public void initialize(InputSplit split, TaskAttemptContext context) throws IOException,
        InterruptedException {

        FileSplit fSplit = (FileSplit) split;
        Path path = fSplit.getPath();
        Configuration conf = context.getConfiguration();
        this.in = new RCFile.Reader(path.getFileSystem(conf), path, conf);
        this.end = fSplit.getStart() + fSplit.getLength();

        if (fSplit.getStart() > in.getPosition()) {
            in.sync(fSplit.getStart());
        }

        this.start = in.getPosition();
        more = start < end;
View Full Code Here
258
259
260
261
262
263
264
265
266
267
268
        long pos = 0;
        int n;
        try {
          while ((n = reader.readLine(key)) > 0) {
            String[] hosts = getStoreDirHosts(fs, path);
            splits.add(new FileSplit(path, pos, n, hosts));
            pos += n;
          }
        } finally {
          reader.close();
        }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.lib.input.FileSplit

  • com.asakusafw.runtime.stage.input.TemporaryInputFormat
  • com.asakusafw.runtime.stage.input.TemporaryInputFormatTest
  • com.chine.kmeans.mapreduce.dataprep.DataPrepMapper
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleInputFormat$TupleInputReader
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat$TupleTextInputReader
  • com.hadoop.mapreduce.LzoLineRecordReader
  • com.hadoop.mapreduce.LzoSplitRecordReader
  • com.hadoop.mapreduce.LzoTextInputFormat
  • com.mongodb.hadoop.splitter.BSONSplitter
  • com.thinkaurelius.faunus.formats.graphson.GraphSONRecordReaderTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.