Package eu.stratosphere.core.fs

Examples of eu.stratosphere.core.fs.FileInputSplit

588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
   
    if (!(split instanceof FileInputSplit)) {
      throw new IllegalArgumentException("File Input Formats can only be used with FileInputSplits.");
    }
   
    final FileInputSplit fileSplit = (FileInputSplit) split;
   
    this.splitStart = fileSplit.getStart();
    this.splitLength = fileSplit.getLength();

    if (LOG.isDebugEnabled()) {
      LOG.debug("Opening input split " + fileSplit.getPath() + " [" + this.splitStart + "," + this.splitLength + "]");
    }

   
    // open the split in an asynchronous thread
    final InputSplitOpenThread isot = new InputSplitOpenThread(fileSplit, this.openTimeout);
    isot.start();
   
    try {
      this.stream = isot.waitForCompletion();
      // Wrap stream in a extracting (decompressing) stream if file ends with .deflate.
      if(fileSplit.getPath().getName().endsWith(DEFLATE_SUFFIX)) {
        this.stream = new InflaterInputStreamFSInputWrapper(stream);
      }
     
    }
    catch (Throwable t) {
      throw new IOException("Error opening the Input Split " + fileSplit.getPath() +
          " [" + splitStart + "," + splitLength + "]: " + t.getMessage(), t);
    }
   
    // get FSDataInputStream
    if (this.splitStart != 0) {
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
  }

  @Test
  public void testOpen() throws IOException {
    final String myString = "my mocked line 1\nmy mocked line 2\n";
    final FileInputSplit split = createTempFile(myString); 
   
    int bufferSize = 5;
    format.setBufferSize(bufferSize);
    format.open(split);
    assertEquals(0, format.splitStart);
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
  }

  @Test
  public void testRead() throws IOException {
    final String myString = "my key|my val$$$my key2\n$$ctd.$$|my value2";
    final FileInputSplit split = createTempFile(myString);
   
    final Configuration parameters = new Configuration();
   
    format.setDelimiter("$$$");
    format.configure(parameters);
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
 
  @Test
  public void testRead2() throws IOException {
    // 2. test case
    final String myString = "my key|my val$$$my key2\n$$ctd.$$|my value2";
    final FileInputSplit split = createTempFile(myString);
   
    final Configuration parameters = new Configuration();
    // default delimiter = '\n'
   
    format.configure(parameters);
View Full Code Here
185
186
187
188
189
190
191
   
    OutputStreamWriter wrt = new OutputStreamWriter(new FileOutputStream(this.tempFile));
    wrt.write(contents);
    wrt.close();
   
    return new FileInputSplit(0, new Path(this.tempFile.toURI().toString()), 0, this.tempFile.length(), new String[] {"localhost"});
  }
View Full Code Here
67
68
69
70
71
72
73
74
    final FileSystem fs = FileSystem.get(normalizedPath.toUri());
    FileStatus fileStatus = fs.getFileStatus(normalizedPath);

    BlockLocation[] blocks = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
    inputFormat.open(new FileInputSplit(0, new Path(path), 0, fileStatus.getLen(), blocks[0].getHosts()));
    return inputFormat;
  }
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
        // get the block locations and make sure they are in order with respect to their offset
        final BlockLocation[] blocks = fs.getFileBlockLocations(file, pos, remainingLength);
        Arrays.sort(blocks);

        inputSplits.add(new FileInputSplit(inputSplits.size(), file.getPath(), pos, remainingLength,
          blocks[0].getHosts()));
      }
    }

    if (inputSplits.size() < minNumSplits) {
      LOG.warn(String.format(
        "With the given block size %d, the file %s cannot be split into %d blocks. Filling up with empty splits...",
        blockSize, this.filePath, minNumSplits));
      FileStatus last = files.get(files.size() - 1);
      final BlockLocation[] blocks = fs.getFileBlockLocations(last, 0, last.getLen());
      for (int index = files.size(); index < minNumSplits; index++) {
        inputSplits.add(new FileInputSplit(index, last.getPath(), last.getLen(), 0, blocks[0].getHosts()));
      }
    }

    return inputSplits.toArray(new FileInputSplit[0]);
  }
View Full Code Here
377
378
379
380
381
382
383
384
385
386
387
      // take the samples
      while (samplesTaken < numSamples && fileNum < allFiles.size()) {
        // make a split for the sample and use it to read a record
        FileStatus file = allFiles.get(fileNum);
        FileInputSplit split = new FileInputSplit(0, file.getPath(), offset, file.getLen() - offset, null);

        // we open the split, read one line, and take its length
        try {
          open(split);
          if (readLine()) {
View Full Code Here
TOP

Related Classes of eu.stratosphere.core.fs.FileInputSplit

  • eu.stratosphere.api.common.io.BinaryInputFormat
  • eu.stratosphere.api.common.io.DelimitedInputFormat
  • eu.stratosphere.api.common.io.DelimitedInputFormatTest
  • eu.stratosphere.api.common.io.FileInputFormat
  • eu.stratosphere.api.common.io.FormatUtil
  • eu.stratosphere.api.common.io.GenericCsvInputFormatTest
  • eu.stratosphere.api.java.io.CsvInputFormatTest
  • eu.stratosphere.api.java.record.io.CsvInputFormatTest
  • eu.stratosphere.api.java.record.io.FixedLenghtInputFormatTest
  • eu.stratosphere.nephele.jobmanager.DoubleSourceTask
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.