Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FSDataInputStream.seek()

238
239
240
241
242
243
244
245
246
247
248
    FileSystem fs = FileSystem.get(uri, conf);
    FSDataInputStream in = null;
    try {
      in = fs.open(new Path(filename));
      if (fileOffset > 0) {
        in.seek(fileOffset);
      }
      in.readFully(buf, bufOffset, len);
    } finally {
      IOUtils.closeStream(in);
    }
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
      end = Long.MAX_VALUE;
    } else {
      if (start != 0) {
        skipFirstLine = true;
        --start;
        fileIn.seek(start);
      }
      in = new LineReader(fileIn, job);
    }
    if (skipFirstLine) {  // skip first line and re-establish "start".
      start += in.readLine(new Text());
View Full Code Here
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
         * send it to the reducer.
         */
        //open the map-output file
        mapOutputIn = fileSys.open(mapOutputFileName);
        //seek to the correct offset for the reduce
        mapOutputIn.seek(startOffset);
         
        long totalRead = 0;
        int len = mapOutputIn.read(buffer, 0,
                                   partLength < MAX_BYTES_TO_READ
                                   ? (int)partLength : MAX_BYTES_TO_READ);
View Full Code Here
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
        for (int parts = 0; parts < partitions; parts++){
          List<SegmentDescriptor> segmentList =
            new ArrayList<SegmentDescriptor>(numSpills);
          for(int i = 0; i < numSpills; i++) {
            FSDataInputStream indexIn = localFs.open(indexFileName[i]);
            indexIn.seek(parts * 16);
            long segmentOffset = indexIn.readLong();
            long segmentLength = indexIn.readLong();
            indexIn.close();
            SegmentDescriptor s = sorter.new SegmentDescriptor(segmentOffset,
                                                               segmentLength, filename[i]);
View Full Code Here
210
211
212
213
214
215
216
217
218
219
220
          String key = file.readUTF();
          String value = file.readUTF();
          opts.put(key, value);
        }
      } else {
        file.seek(0);
        return file;
      }
      return file;
    } catch (IOException ex) {
      file.seek(0);
View Full Code Here
215
216
217
218
219
220
221
222
223
224
225
        file.seek(0);
        return file;
      }
      return file;
    } catch (IOException ex) {
      file.seek(0);
      return file;
    }
  }

  @SuppressWarnings("deprecation")
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
    else
    {
      if (start != 0) //TODO: start is always zero?
      {
            --start;
            _input.seek(start);
      }
      internalInputStream = inputStream = new CountingInputStream(_input, start);
    }
    Collection rtr = CollectionFactory.loadCollection(
      ApplicationSetup.getProperty("trec.collection.class", "TRECCollection"),
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
          inputs[i] = new FSDataInputStream(new RaidUtils.ZeroInputStream(
              offset + blockSize));
        } else {
          FSDataInputStream in = parityFs.open(
              parityFile, conf.getInt("io.file.buffer.size", 64 * 1024));
          in.seek(offset);
          LOG.info("Adding " + parityFile + ":" + offset + " as input " + i);
          inputs[i] = in;
        }
      }
      // Now open streams to the data blocks.
View Full Code Here
164
165
166
167
168
169
170
171
172
173
174
          inputs[i] = new FSDataInputStream(new RaidUtils.ZeroInputStream(
              offset + blockSize));
        } else {
          FSDataInputStream in = fs.open(
            srcFile, conf.getInt("io.file.buffer.size", 64 * 1024));
          in.seek(offset);
          LOG.info("Adding " + srcFile + ":" + offset + " as input " + i);
          inputs[i] = in;
        }
      }
      if (erasedLocations.size() > paritySize) {
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
    try {
      for (int i = 0; i < codec.stripeLength; i++) {
        long seekOffset = stripeStartOffset + i * blockSize;
        if (seekOffset < srcSize) {
          FSDataInputStream in = fs.open(srcFile, bufferSize);
          in.seek(seekOffset);
          LOG.info("Opening stream at " + srcFile + ":" + seekOffset);
          blocks[i] = in;
        } else {
          LOG.info("Using zeros at offset " + seekOffset);
          // We have no src data at this offset.
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.