Package org.apache.hadoop.io.compress.lz4

Examples of org.apache.hadoop.io.compress.lz4.Lz4Decompressor

172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
  // test Lz4Compressor compressor.compress() 
  @Test
  public void testSetInputWithBytesSizeMoreThenDefaultLz4CompressorByfferSize() {
    int BYTES_SIZE = 1024 * 64 + 1;
    try {
      Lz4Compressor compressor = new Lz4Compressor();
      byte[] bytes = generate(BYTES_SIZE);
      assertTrue("needsInput error !!!", compressor.needsInput());
      compressor.setInput(bytes, 0, bytes.length);
      byte[] emptyBytes = new byte[BYTES_SIZE];
      int csize = compressor.compress(emptyBytes, 0, bytes.length);
      assertTrue(
          "testSetInputWithBytesSizeMoreThenDefaultLz4CompressorByfferSize error !!!",
          csize != 0);
    } catch (Exception ex) {
      fail("testSetInputWithBytesSizeMoreThenDefaultLz4CompressorByfferSize ex error !!!");
View Full Code Here
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
  // test compress/decompress process
  @Test
  public void testCompressDecompress() {
    int BYTE_SIZE = 1024 * 54;
    byte[] bytes = generate(BYTE_SIZE);
    Lz4Compressor compressor = new Lz4Compressor();
    try {
      compressor.setInput(bytes, 0, bytes.length);
      assertTrue("Lz4CompressDecompress getBytesRead error !!!",
          compressor.getBytesRead() > 0);
      assertTrue(
          "Lz4CompressDecompress getBytesWritten before compress error !!!",
          compressor.getBytesWritten() == 0);

      byte[] compressed = new byte[BYTE_SIZE];
      int cSize = compressor.compress(compressed, 0, compressed.length);
      assertTrue(
          "Lz4CompressDecompress getBytesWritten after compress error !!!",
          compressor.getBytesWritten() > 0);
      Lz4Decompressor decompressor = new Lz4Decompressor();
      // set as input for decompressor only compressed data indicated with cSize
      decompressor.setInput(compressed, 0, cSize);
      byte[] decompressed = new byte[BYTE_SIZE];
      decompressor.decompress(decompressed, 0, decompressed.length);

      assertTrue("testLz4CompressDecompress finished error !!!", decompressor.finished());     
      assertArrayEquals(bytes, decompressed);
      compressor.reset();
      decompressor.reset();
      assertTrue("decompressor getRemaining error !!!",decompressor.getRemaining() == 0);
    } catch (Exception e) {
      fail("testLz4CompressDecompress ex error!!!");
    }
View Full Code Here
232
233
234
235
236
237
238
239
240
241
242
    BlockDecompressorStream blockDecompressorStream = null;
    try {
      // compress empty stream
      bytesOut = new ByteArrayOutputStream();
      BlockCompressorStream blockCompressorStream = new BlockCompressorStream(
          bytesOut, new Lz4Compressor(), 1024, 0);
      // close without write
      blockCompressorStream.close();
      // check compressed output
      buf = bytesOut.toByteArray();
      assertEquals("empty stream compressed output size != 4", 4, buf.length);
View Full Code Here
271
272
273
274
275
276
277
278
279
280
281
    int bufferSize = 262144;
    int compressionOverhead = (bufferSize / 6) + 32;
    try {
      DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
      CompressionOutputStream deflateFilter = new BlockCompressorStream(
          compressedDataBuffer, new Lz4Compressor(bufferSize), bufferSize,
          compressionOverhead);
      deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter));
      deflateOut.write(bytes, 0, bytes.length);
      deflateOut.flush();
      deflateFilter.finish();
View Full Code Here
45
46
47
48
49
50
51
    }
    int bufferSize = conf.getInt(
        IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
        IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
    System.out.println("Create Lz4hc codec");
    return new Lz4Compressor(bufferSize, true);
  }
View Full Code Here
152
153
154
155
156
157
158
        IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
        IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
    boolean useLz4HC = conf.getBoolean(
        IO_COMPRESSION_CODEC_LZ4_USELZ4HC_KEY,
        IO_COMPRESSION_CODEC_LZ4_USELZ4HC_DEFAULT);
    return new Lz4Compressor(bufferSize, useLz4HC);
  }
View Full Code Here
204
205
206
207
208
209
210
      throw new RuntimeException("native lz4 library not available");
    }
    int bufferSize = conf.getInt(
        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
    return new Lz4Decompressor(bufferSize);
  }
View Full Code Here
207
208
209
210
211
212
213
      throw new RuntimeException("native lz4 library not available");
    }
    int bufferSize = conf.getInt(
        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
    return new Lz4Decompressor(bufferSize);
  }
View Full Code Here
200
201
202
203
204
205
206
      throw new RuntimeException("native lz4 library not available");
    }
    int bufferSize = conf.getInt(
        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY,
        CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT);
    return new Lz4Decompressor(bufferSize);
  }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
   
    byte[] rawData = generate(SIZE);
    try {
      CompressDecompressTester.of(rawData)
          .withCompressDecompressPair(new SnappyCompressor(), new SnappyDecompressor())
          .withCompressDecompressPair(new Lz4Compressor(), new Lz4Decompressor())
          .withCompressDecompressPair(new BuiltInZlibDeflater(), new BuiltInZlibInflater())
          .withTestCases(ImmutableSet.of(CompressionTestStrategy.COMPRESS_DECOMPRESS_SINGLE_BLOCK,
                      CompressionTestStrategy.COMPRESS_DECOMPRESS_BLOCK,
                      CompressionTestStrategy.COMPRESS_DECOMPRESS_ERRORS,
                      CompressionTestStrategy.COMPRESS_DECOMPRESS_WITH_EMPTY_STREAM))
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.lz4.Lz4Decompressor

  • io.druid.client.cache.LZ4Transcoder
  • org.apache.hadoop.io.compress.BlockCompressorStream
  • org.apache.hadoop.io.compress.BlockDecompressorStream
  • org.apache.hadoop.io.compress.CompressionInputStream
  • org.apache.hadoop.io.compress.CompressionOutputStream
  • org.apache.hadoop.io.compress.lz4.Lz4Compressor
  • org.apache.hadoop.io.compress.lz4.Lz4Decompressor
  • org.apache.hadoop.io.compress.lz4.TestLz4CompressorDecompressor
  • org.apache.hadoop.io.compress.Lz4Codec
  • org.apache.hadoop.io.compress.Lz4hcCodec
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.