Examples of Decompressor


Examples of org.apache.hadoop.io.compress.Decompressor

     * @throws IOException
     */
    private ByteBuffer decompress(final long offset, final int compressedSize,
      final int decompressedSize, final boolean pread)
    throws IOException {
      Decompressor decompressor = null;
      ByteBuffer buf = null;
      try {
        decompressor = this.compressAlgo.getDecompressor();
        // My guess is that the bounded range fis is needed to stop the
        // decompressor reading into next block -- IIRC, it just grabs a
View Full Code Here

Examples of org.apache.hadoop.io.compress.Decompressor

          "Output buffer does not have enough space to hold "
              + uncompressedSize + " decompressed bytes, available: "
              + (dest.length - destOffset));
    }

    Decompressor decompressor = null;
    try {
      decompressor = compressAlgo.getDecompressor();
      InputStream is = compressAlgo.createDecompressionStream(
          bufferedBoundedStream, decompressor, 0);
View Full Code Here

Examples of org.apache.hadoop.io.compress.Decompressor

          "Output buffer does not have enough space to hold "
              + uncompressedSize + " decompressed bytes, available: "
              + (dest.length - destOffset));
    }

    Decompressor decompressor = null;
    try {
      decompressor = compressAlgo.getDecompressor();
      InputStream is = compressAlgo.createDecompressionStream(
          bufferedBoundedStream, decompressor, 0);
View Full Code Here

Examples of org.apache.hadoop.io.compress.Decompressor

    IFileInputStream checksumIn = new IFileInputStream(input, compressedLength,
        ifileReadAhead, ifileReadAheadLength);

    input = checksumIn;

    Decompressor decompressor = null;
   
    // Are map-outputs compressed?
    if (codec != null) {
      decompressor = CodecPool.getDecompressor(codec);
      decompressor.reset();
      input = codec.createInputStream(input, decompressor);
    }
    // Copy map-output into an in-memory buffer
    byte[] shuffleData = fetchedInput.getBytes();

    try {
      IOUtils.readFully(input, shuffleData, 0, shuffleData.length);
      // metrics.inputBytes(shuffleData.length);
      LOG.info("Read " + shuffleData.length + " bytes from input for "
          + fetchedInput.getInputAttemptIdentifier());
    } catch (IOException ioe) {
      // Close the streams
      IOUtils.cleanup(LOG, input);
      // Re-throw
      throw ioe;
    } finally {
      if(decompressor != null) {
        decompressor.reset();
        CodecPool.returnDecompressor(decompressor);
      }
    }
  }
View Full Code Here

Examples of org.apache.hadoop.io.compress.Decompressor

        int bufferSize, long length) throws IOException {
      return fs.open(file, bufferSize);
    }
   
    private Decompressor getPooledOrNewDecompressor() {
      Decompressor decompressor = null;
      decompressor = decompressorPool.getCodec(codec.getDecompressorType());
      if (decompressor == null) {
        decompressor = codec.createDecompressor();
      }
      return decompressor;
View Full Code Here

Examples of org.apache.hadoop.io.compress.Decompressor

   *          <code>Decompressor</code>
   * @return <code>Decompressor</code> for the given
   *         <code>CompressionCodec</code> the pool or a new one
   */
  public static Decompressor getDecompressor(CompressionCodec codec) {
    Decompressor decompressor = borrow(DECOMPRESSOR_POOL, codec
        .getDecompressorType());
    if (decompressor == null) {
      decompressor = codec.createDecompressor();
      LOG.info("Got brand-new decompressor");
    } else {
View Full Code Here

Examples of org.apache.hadoop.io.compress.Decompressor

    // resources if the stream is not closed properly after we let it out.
    InputStream is = null;
    if (compressor != null) {
      // GZIPCodec fails w/ NPE if no configuration.
      if (compressor instanceof Configurable) ((Configurable)compressor).setConf(this.conf);
      Decompressor poolDecompressor = CodecPool.getDecompressor(compressor);
      CompressionInputStream cis =
        compressor.createInputStream(new ByteArrayInputStream(cellBlock, offset, length),
        poolDecompressor);
      ByteBufferOutputStream bbos = null;
      try {
View Full Code Here

Examples of org.apache.hadoop.io.compress.Decompressor

          "Output buffer does not have enough space to hold "
              + uncompressedSize + " decompressed bytes, available: "
              + (dest.length - destOffset));
    }

    Decompressor decompressor = null;
    try {
      decompressor = compressAlgo.getDecompressor();
      InputStream is = compressAlgo.createDecompressionStream(
          bufferedBoundedStream, decompressor, 0);
View Full Code Here

Examples of org.apache.hadoop.io.compress.Decompressor

     * @throws IOException
     */
    protected void decompress(byte[] dest, int destOffset,
        InputStream bufferedBoundedStream,
        int uncompressedSize) throws IOException {
      Decompressor decompressor = null;
      try {
        decompressor = compressAlgo.getDecompressor();
        InputStream is = compressAlgo.createDecompressionStream(
            bufferedBoundedStream, decompressor, 0);

View Full Code Here

Examples of org.apache.hadoop.io.compress.Decompressor

    FileSystem fs = FileSystem.get(conf);
    InputStream is = fs.open(f);
    CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
    CompressionCodec codec = ccf.getCodec(f);
    LOG.info("gzip check codec is " + codec);
    Decompressor decompressor = CodecPool.getDecompressor(codec);
    if (null == decompressor) {
      LOG.info("Verifying gzip sanity with null decompressor");
    } else {
      LOG.info("Verifying gzip sanity with decompressor: "
          + decompressor.toString());
    }
    is = codec.createInputStream(is, decompressor);
    BufferedReader r = new BufferedReader(new InputStreamReader(is));
    int numLines = 0;
    while (true) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.