Examples of org.apache.hadoop.fs.FSDataInputStream.seek()

Class org.apache.hadoop.fs.FSDataInputStream

Examples of org.apache.hadoop.fs.FSDataInputStream.seek()

org.apache.hadoop.fs.FSDataInputStream.seek()
Seek to the given offset. @param desired offset to seek to


    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    if (start != 0) {
        fileIn.seek(start);
    }
    in = new BufferedPositionedInputStream(fileIn, start);
    inData = new DataInputStream(in);
  }

View Full Code Here

         * send it to the reducer.
         */
        //open the map-output file
        mapOutputIn = fileSys.open(mapOutputFileName);
        //seek to the correct offset for the reduce
        mapOutputIn.seek(startOffset);
          
        long totalRead = 0;
        int len = mapOutputIn.read(buffer, 0,
                                   partLength < MAX_BYTES_TO_READ 
                                   ? (int)partLength : MAX_BYTES_TO_READ);

View Full Code Here

    this.fileSystem = fs;
    this.path = path;
    FSDataInputStream file = fs.open(path);
    long size = fs.getFileStatus(path).getLen();
    int readSize = (int) Math.min(size, DIRECTORY_SIZE_GUESS);
    file.seek(size - readSize);
    ByteBuffer buffer = ByteBuffer.allocate(readSize);
    file.readFully(buffer.array(), buffer.arrayOffset() + buffer.position(),
      buffer.remaining());
    int psLen = buffer.get(readSize - 1);
    ensureOrcFooter(file, path, psLen, buffer);

View Full Code Here

        throw new IllegalArgumentException("Unknown compression");
    }
    codec = WriterImpl.createCodec(compressionKind);
    int extra = Math.max(0, psLen + 1 + footerSize - readSize);
    if (extra > 0) {
      file.seek(size - readSize - extra);
      ByteBuffer extraBuf = ByteBuffer.allocate(extra + readSize);
      file.readFully(extraBuf.array(),
        extraBuf.arrayOffset() + extraBuf.position(), extra);
      extraBuf.position(extra);
      extraBuf.put(buffer);

View Full Code Here

    FileSystem fs = FileSystem.get(uri, conf);
    FSDataInputStream in = null;
    try {
      in = fs.open(new Path(filename));
      if (fileOffset > 0) {
        in.seek(fileOffset);
      }
      in.readFully(buf, bufOffset, len);
    } finally {
      IOUtils.closeStream(in);
    }

View Full Code Here

      in = codec.createInputStream(fileIn);
      end = Long.MAX_VALUE;
    } else if (start != 0) {
      skipFirstLine = true;  // wait till BufferedInputStream to skip
      --start;
      fileIn.seek(start);
    }


    this.in = new BufferedInputStream(in);
    if (skipFirstLine) {  // skip first line and re-establish "start".
      start += LineRecordReader.readLine(this.in, null);

View Full Code Here

         * send it to the reducer.
         */
        //open the map-output file
        mapOutputIn = fileSys.open(mapOutputFileName);
        //seek to the correct offset for the reduce
        mapOutputIn.seek(startOffset);
          
        long totalRead = 0;
        int len = mapOutputIn.read(buffer, 0,
                                   partLength < MAX_BYTES_TO_READ 
                                   ? (int)partLength : MAX_BYTES_TO_READ);

View Full Code Here

        for (int parts = 0; parts < partitions; parts++){
          List<SegmentDescriptor> segmentList =
            new ArrayList<SegmentDescriptor>(numSpills);
          for(int i = 0; i < numSpills; i++) {
            FSDataInputStream indexIn = localFs.open(indexFileName[i]);
            indexIn.seek(parts * 16);
            long segmentOffset = indexIn.readLong();
            long segmentLength = indexIn.readLong();
            indexIn.close();
            SegmentDescriptor s = sorter.new SegmentDescriptor(segmentOffset,
                                                               segmentLength, filename[i]);

View Full Code Here

         */
        //open the map-output file
        mapOutputIn = rfs.open(mapOutputFileName);


        //seek to the correct offset for the reduce
        mapOutputIn.seek(info.startOffset);
        long rem = info.partLength;
        int len =
          mapOutputIn.read(buffer, 0, (int)Math.min(rem, MAX_BYTES_TO_READ));
        while (rem > 0 && len >= 0) {
          rem -= len;

View Full Code Here

        for (int parts = 0; parts < partitions; parts++){
          List<SegmentDescriptor> segmentList =
            new ArrayList<SegmentDescriptor>(numSpills);
          for(int i = 0; i < numSpills; i++) {
            FSDataInputStream indexIn = localFs.open(indexFileName[i]);
            indexIn.seek(parts * 16);
            long segmentOffset = indexIn.readLong();
            long segmentLength = indexIn.readLong();
            indexIn.close();
            SegmentDescriptor s = sorter.new SegmentDescriptor(segmentOffset,
                                                               segmentLength, filename[i]);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.