Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.BlockCompressorStream

142
143
144
145
146
147
148
149
    LzoCompressor.CompressionStrategy strategy = getCompressionStrategy(conf);
    int bufferSize = getBufferSize(conf);
    int compressionOverhead = strategy.name().contains("LZO1") ?
        (bufferSize >> 4) + 64 + 3 : (bufferSize >> 3) + 128 + 3;

    return new BlockCompressorStream(out, compressor, bufferSize,
        compressionOverhead);
  }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
      di.readFully(data);
      LOG.info("Original data is " + data.length + " bytes.");

      ByteArrayInputStream bi = new ByteArrayInputStream(data);
      ByteArrayOutputStream bo = new ByteArrayOutputStream(data.length);
      BlockCompressorStream co = new BlockCompressorStream(bo, new BuiltInZlibDeflater());
      LOG.info("Starting.");
      long start = System.currentTimeMillis();
      IOUtils.copy(bi, co);
      co.close();
      long end = System.currentTimeMillis();
      LOG.info("Compression took " + ((end - start) / 1000d) + " ms");
      LOG.info("Compressed data is " + bo.size() + " bytes.");

      if (true)
View Full Code Here
139
140
141
142
143
144
145
146
        int bufferSize = getBufferSize(conf);
        int compressionOverhead = strategy.name().contains("LZO1")
                ? (bufferSize >> 4) + 64 + 3
                : (bufferSize >> 3) + 128 + 3;

        return new BlockCompressorStream(out, compressor, bufferSize,
                compressionOverhead);
    }
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
            LOG.info("Original data is " + data.length + " bytes.");

            for (int i = 0; i < 1; i++) {
                ByteArrayInputStream bi = new ByteArrayInputStream(data);
                ByteArrayOutputStream bo = new ByteArrayOutputStream(data.length);
                BlockCompressorStream co = new BlockCompressorStream(bo, new LzoCompressor(), 64 * 1024, 18);
                LOG.info("Starting.");
                long start = System.currentTimeMillis();
                IOUtils.copy(bi, co);
                co.close();
                long end = System.currentTimeMillis();
                LOG.info("Compression took " + ((end - start) / 1000d) + " ms");
                LOG.info("Compressed data is " + bo.size() + " bytes.");

                byte[] cb = bo.toByteArray();
View Full Code Here
209
210
211
212
213
214
215
216
217
218
219
220
221
222
    byte[] buf = null;
    BlockDecompressorStream blockDecompressorStream = null;
    try {
      // compress empty stream
      bytesOut = new ByteArrayOutputStream();
      BlockCompressorStream blockCompressorStream = new BlockCompressorStream(
          bytesOut, new SnappyCompressor(), 1024, 0);
      // close without write
      blockCompressorStream.close();

      // check compressed output
      buf = bytesOut.toByteArray();
      assertEquals("empty stream compressed output size != 4", 4, buf.length);
View Full Code Here
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
    int compressionOverhead = (bufferSize / 6) + 32;
    DataOutputStream deflateOut = null;
    DataInputStream inflateIn = null;
    try {
      DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
      CompressionOutputStream deflateFilter = new BlockCompressorStream(
          compressedDataBuffer, new SnappyCompressor(bufferSize), bufferSize,
          compressionOverhead);
      deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter));

      deflateOut.write(bytes, 0, bytes.length);
      deflateOut.flush();
      deflateFilter.finish();

      DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
      deCompressedDataBuffer.reset(compressedDataBuffer.getData(), 0,
          compressedDataBuffer.getLength());
View Full Code Here
231
232
233
234
235
236
237
238
239
240
241
242
243
244
    byte[] buf = null;
    BlockDecompressorStream blockDecompressorStream = null;
    try {
      // compress empty stream
      bytesOut = new ByteArrayOutputStream();
      BlockCompressorStream blockCompressorStream = new BlockCompressorStream(
          bytesOut, new Lz4Compressor(), 1024, 0);
      // close without write
      blockCompressorStream.close();
      // check compressed output
      buf = bytesOut.toByteArray();
      assertEquals("empty stream compressed output size != 4", 4, buf.length);
      // use compressed output as input for decompression
      bytesIn = new ByteArrayInputStream(buf);
View Full Code Here
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
    byte[] bytes = generate(BYTE_SIZE);
    int bufferSize = 262144;
    int compressionOverhead = (bufferSize / 6) + 32;
    try {
      DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
      CompressionOutputStream deflateFilter = new BlockCompressorStream(
          compressedDataBuffer, new Lz4Compressor(bufferSize), bufferSize,
          compressionOverhead);
      deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter));
      deflateOut.write(bytes, 0, bytes.length);
      deflateOut.flush();
      deflateFilter.finish();

      DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
      deCompressedDataBuffer.reset(compressedDataBuffer.getData(), 0,
          compressedDataBuffer.getLength());
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.BlockCompressorStream

  • com.hadoop.compression.lzo.LzoCodec
  • org.anarres.lzo.BlockCompressorStreamTest
  • org.anarres.lzo.hadoop.codec.LzoCodec
  • org.anarres.lzo.ZlibCompressorTest
  • org.apache.hadoop.io.compress.lz4.TestLz4CompressorDecompressor
  • org.apache.hadoop.io.compress.snappy.TestSnappyCompressorDecompressor
  • java.io.IOException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.