Examples of FSDataInputStreamWrapper

eu.stratosphere.api.avro.FSDataInputStreamWrapper
Code copy pasted from org.apache.avro.mapred.FSInput (which is Apache licensed as well) The wrapper keeps track of the position in the data stream.
org.apache.flink.api.avro.FSDataInputStreamWrapper
Code copy pasted from org.apache.avro.mapred.FSInput (which is Apache licensed as well) The wrapper keeps track of the position in the data stream.
org.apache.hadoop.hbase.io.FSDataInputStreamWrapper
Wrapper for input stream(s) that takes care of the interaction of FS and HBase checksums, as well as closing streams. Initialization is not thread-safe, but normal operation is; see method comments.

Examples of eu.stratosphere.api.avro.FSDataInputStreamWrapper

      datumReader = new ReflectDatumReader<E>(avroValueType);
    }
    
    LOG.info("Opening split " + split);
    
    SeekableInput in = new FSDataInputStreamWrapper(stream, (int) split.getLength());
    
    dataFileReader = DataFileReader.openReader(in, datumReader);
    dataFileReader.sync(split.getStart());
  }

View Full Code Here

Examples of org.apache.flink.api.avro.FSDataInputStreamWrapper

      datumReader = new ReflectDatumReader<E>(avroValueType);
    }
    
    LOG.info("Opening split " + split);
    
    SeekableInput in = new FSDataInputStreamWrapper(stream, (int) split.getLength());
    
    dataFileReader = DataFileReader.openReader(in, datumReader);
    dataFileReader.sync(split.getStart());
  }

View Full Code Here

Examples of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper

   * This factory method is used only by unit tests
   */
  static Reader createReaderFromStream(Path path,
      FSDataInputStream fsdis, long size, CacheConfig cacheConf, Configuration conf)
      throws IOException {
    FSDataInputStreamWrapper wrapper = new FSDataInputStreamWrapper(fsdis);
    return pickReaderVersion(path, wrapper, size, cacheConf, null, conf);
  }

View Full Code Here

Examples of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper

   * @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile is corrupt/invalid.
   */
  public static Reader createReader(
      FileSystem fs, Path path, CacheConfig cacheConf, Configuration conf) throws IOException {
    Preconditions.checkNotNull(cacheConf, "Cannot create Reader with null CacheConf");
    FSDataInputStreamWrapper stream = new FSDataInputStreamWrapper(fs, path);
    return pickReaderVersion(path, stream, fs.getFileStatus(path).getLen(),
      cacheConf, stream.getHfs(), conf);
  }

View Full Code Here

Examples of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper

   */
  public static Reader createReaderWithEncoding(
      FileSystem fs, Path path, CacheConfig cacheConf,
      DataBlockEncoding preferredEncodingInCache) throws IOException {
    final boolean closeIStream = true;
    FSDataInputStreamWrapper stream = new FSDataInputStreamWrapper(fs, path);
    return pickReaderVersion(path, stream, fs.getFileStatus(path).getLen(),
        cacheConf, preferredEncodingInCache, stream.getHfs());
  }

View Full Code Here

Examples of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper

   * This factory method is used only by unit tests
   */
  static Reader createReaderFromStream(Path path,
      FSDataInputStream fsdis, long size, CacheConfig cacheConf)
      throws IOException {
    FSDataInputStreamWrapper wrapper = new FSDataInputStreamWrapper(fsdis);
    return pickReaderVersion(path, wrapper, size, cacheConf, DataBlockEncoding.NONE, null);
  }

View Full Code Here

Examples of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper

          totalSize += hbw.getOnDiskSizeWithHeader();
        }
        os.close();


        FSDataInputStream is = fs.open(path);
        HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(is),
            algo, totalSize, MINOR_VERSION, fs, path);
        HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
        is.close();


        b.sanityCheck();
        assertEquals(4936, b.getUncompressedSizeWithoutHeader());
        assertEquals(algo == GZ ? 2173 : 4936,
                     b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
        String blockStr = b.toString();


        if (algo == GZ) {
          is = fs.open(path);
          hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(is),
              algo, totalSize, MINOR_VERSION, fs, path);
          b = hbr.readBlockData(0, 2173 + HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM +
                                b.totalChecksumBytes(), -1, pread);
          assertEquals(blockStr, b.toString());
          int wrongCompressedSize = 2172;

View Full Code Here

Examples of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper

            totalSize += hbw.getOnDiskSizeWithHeader();
          }
          os.close();


          FSDataInputStream is = fs.open(path);
          HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(is),
              algo, totalSize, MINOR_VERSION, fs, path);
          hbr.setDataBlockEncoder(dataBlockEncoder);
          hbr.setIncludesMemstoreTS(includesMemstoreTS);


          HFileBlock b;

View Full Code Here

Examples of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper

   * @param dataBlockEncoding data block encoding algorithm.
   * @return The StoreFile.Reader for the file
   */
  public StoreFile.Reader open(final FileSystem fs, final CacheConfig cacheConf,
      final DataBlockEncoding dataBlockEncoding) throws IOException {
    FSDataInputStreamWrapper in;
    FileStatus status;


    if (this.link != null) {
      // HFileLink
      in = new FSDataInputStreamWrapper(fs, this.link);
      status = this.link.getFileStatus(fs);
    } else if (this.reference != null) {
      // HFile Reference
      Path referencePath = getReferredToFile(this.getPath());
      in = new FSDataInputStreamWrapper(fs, referencePath);
      status = fs.getFileStatus(referencePath);
    } else {
      in = new FSDataInputStreamWrapper(fs, this.getPath());
      status = fileStatus;
    }
    long length = status.getLen();
    if (this.reference != null) {
      hdfsBlocksDistribution = computeRefFileHDFSBlockDistribution(fs, reference, status);

View Full Code Here

Examples of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper

   * @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile is corrupt/invalid.
   */
  public static Reader createReader(
      FileSystem fs, Path path, CacheConfig cacheConf) throws IOException {
    Preconditions.checkNotNull(cacheConf, "Cannot create Reader with null CacheConf");
    FSDataInputStreamWrapper stream = new FSDataInputStreamWrapper(fs, path);
    return pickReaderVersion(path, stream, fs.getFileStatus(path).getLen(), cacheConf, null);
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.