Examples of org.apache.hadoop.io.compress.Compressor

org.apache.hadoop.io.compress.Compressor
Specification of a stream-based 'compressor' which can be plugged into a {@link CompressionOutputStream} to compress data.This is modelled after {@link java.util.zip.Deflater}

    conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, true);
    if (ZlibFactory.isNativeZlibLoaded(conf)) {
      byte[] rawData;
      int tryNumber = 5;
      int BYTE_SIZE = 10 * 1024;
      Compressor zlibCompressor = ZlibFactory.getZlibCompressor(conf);
      Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf);
      rawData = generate(BYTE_SIZE);
      try {
        for (int i = 0; i < tryNumber; i++)
          compressDecompressZlib(rawData, (ZlibCompressor) zlibCompressor,
              (ZlibDecompressor) zlibDecompressor);
        zlibCompressor.reinit(conf);
      } catch (Exception ex) {
        fail("testZlibCompressorDecompressorWithConfiguration ex error " + ex);
      }
    } else {
      assertTrue("ZlibFactory is using native libs against request",

View Full Code Here

  @Test
  public void testZlibCompressorDecompressorSetDictionary() {
    Configuration conf = new Configuration();
    conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, true);
    if (ZlibFactory.isNativeZlibLoaded(conf)) {
      Compressor zlibCompressor = ZlibFactory.getZlibCompressor(conf);
      Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf);


      checkSetDictionaryNullPointerException(zlibCompressor);
      checkSetDictionaryNullPointerException(zlibDecompressor);

View Full Code Here

  throws IOException {
    FileSystem outFS = path.getFileSystem(conf);
    CompressionCodec codec =
      new CompressionCodecFactory(conf).getCodec(path);
    OutputStream output;
    Compressor compressor = null;
    if (codec != null) {
      compressor = CodecPool.getCompressor(codec);
      output = codec.createOutputStream(outFS.create(path), compressor);
    } else {
      output = outFS.create(path);

View Full Code Here


    private void flushRecords() throws IOException {


      key.numberRows = bufferedRecords;


      Compressor compressor = null;
      NonSyncDataOutputBuffer valueBuffer = null;
      CompressionOutputStream deflateFilter = null;
      DataOutputStream deflateOut = null;
      boolean isCompressed = isCompressed();
      int valueLength = 0;

View Full Code Here

      checkAndWriteSync(); // sync
      out.writeInt(recordLen); // total record length
      out.writeInt(keyLength); // key portion length


      if(this.isCompressed()) {
        Compressor compressor = CodecPool.getCompressor(codec);
        NonSyncDataOutputBuffer compressionBuffer =
          new NonSyncDataOutputBuffer();
        CompressionOutputStream deflateFilter =
          codec.createOutputStream(compressionBuffer, compressor);
        DataOutputStream deflateOut = new DataOutputStream(deflateFilter);

View Full Code Here

      FileSystem outFS = outPath.getFileSystem(conf);


      CompressionCodec codec =
          new CompressionCodecFactory(conf).getCodec(outPath);
      OutputStream output;
      Compressor compressor = null;
      if (codec != null) {
        compressor = CodecPool.getCompressor(codec);
        output = codec.createOutputStream(outFS.create(outPath), compressor);
      } else {
        output = outFS.create(outPath);

View Full Code Here

            tempPaths.add(nextSegment);


            CompressionCodec codec =
                new CompressionCodecFactory(conf).getCodec(nextSegment);
            OutputStream output;
            Compressor compressor = null;
            if (codec != null) {
              compressor = CodecPool.getCompressor(codec);
              output = codec.createOutputStream(tempUncompOut, compressor);
            } else {
              output = tempUncompOut;

View Full Code Here

    fs = (HFileSystem)HFileSystem.get(TEST_UTIL.getConfiguration());
  }


  public byte[] createTestV1Block(Compression.Algorithm algo)
      throws IOException {
    Compressor compressor = algo.getCompressor();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    OutputStream os = algo.createCompressionStream(baos, compressor, 0);
    DataOutputStream dos = new DataOutputStream(os);
    BlockType.META.write(dos); // Let's make this a meta block.
    TestHFileBlock.writeTestBlockContents(dos);

View Full Code Here

    return totalSize;
  }


  public byte[] createTestV1Block(Compression.Algorithm algo)
      throws IOException {
    Compressor compressor = algo.getCompressor();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    OutputStream os = algo.createCompressionStream(baos, compressor, 0);
    DataOutputStream dos = new DataOutputStream(os);
    BlockType.META.write(dos); // Let's make this a meta block.
    writeTestBlockContents(dos);

View Full Code Here

    return totalSize;
  }


  public byte[] createTestV1Block(Compression.Algorithm algo)
      throws IOException {
    Compressor compressor = algo.getCompressor();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    OutputStream os = algo.createCompressionStream(baos, compressor, 0);
    DataOutputStream dos = new DataOutputStream(os);
    BlockType.META.write(dos); // Let's make this a meta block.
    writeTestBlockContents(dos);

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.hadoop.io.compress.Compressor

cascading.tuple.hadoop.collect.HadoopSpillableTupleList

com.cloudera.flume.handlers.hdfs.CustomDfsSink

com.hadoop.compression.lzo.TestLzoCodec

org.apache.hadoop.hbase.io.hfile.TestHFileBlock

org.apache.hadoop.hbase.io.hfile.TestHFileBlockCompatibility

org.apache.hadoop.hbase.ipc.IPCUtil

org.apache.hadoop.hbase.util.CompressionTest

org.apache.hadoop.hive.ql.io.CodecPool

org.apache.hadoop.hive.ql.io.RCFile$Writer

org.apache.hadoop.io.compress.zlib.TestZlibCompressorDecompressor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.