Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.CompressionInputStream

1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
      compressedKeyLen = in.readInt();
      if (decompress) {
        keyTempBuffer.reset();
        keyTempBuffer.write(in, compressedKeyLen);
        keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen);
        CompressionInputStream deflatFilter = codec.createInputStream(
            keyDecompressBuffer, keyDecompressor);
        DataInputStream compressedIn = new DataInputStream(deflatFilter);
        deflatFilter.resetState();
        keyDecompressedData.reset();
        keyDecompressedData.write(compressedIn, currentKeyLength);
        keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength);
        currentKey.readFields(keyDataIn);
      } else {
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
            }
            if(codec instanceof Configurable) {
                ((Configurable)codec).setConf(new Configuration());
            }

            CompressionInputStream createInputStream = codec
                  .createInputStream(new FileInputStream(file));
            int b;
            StringBuffer sb = new StringBuffer();
            while ((b = createInputStream.read()) != -1) {
               sb.append((char) b);
            }
            createInputStream.close();

            // Assert for the number of fields and keys.
            String[] fields = sb.toString().split("\\t");
            assertEquals(3, fields.length);
            String id = indexFolder.substring(1,2);
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
   * @return
   */
  @Override
  public BSPMessageBundle<M> decompressBundle(BSPCompressedBundle compMsgBundle) {
    ByteArrayInputStream bis = null;
    CompressionInputStream sis = null;
    DataInputStream dis = null;
    BSPMessageBundle<M> bundle = new BSPMessageBundle<M>();

    try {
      byte[] data = compMsgBundle.getData();
      bis = new ByteArrayInputStream(data);
      sis = codec.createInputStream(bis);
      dis = new DataInputStream(sis);

      bundle.readFields(dis);

    } catch (IOException ioe) {
      LOG.error("Unable to decompress.", ioe);
    } finally {
      try {
        dis.close();
        sis.close();
        bis.close();
      } catch (IOException e) {
        LOG.warn("Failed to close decompression streams.", e);
      }
    }
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
    InputStream is = null;
    if (compressor != null) {
      // GZIPCodec fails w/ NPE if no configuration.
      if (compressor instanceof Configurable) ((Configurable)compressor).setConf(this.conf);
      Decompressor poolDecompressor = CodecPool.getDecompressor(compressor);
      CompressionInputStream cis =
        compressor.createInputStream(new ByteArrayInputStream(cellBlock, offset, length),
        poolDecompressor);
      try {
        // TODO: This is ugly.  The buffer will be resized on us if we guess wrong.
        // TODO: Reuse buffers.
View Full Code Here
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
      compressedKeyLen = in.readInt();
      if (decompress) {
        keyTempBuffer.reset();
        keyTempBuffer.write(in, compressedKeyLen);
        keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen);
        CompressionInputStream deflatFilter = codec.createInputStream(
            keyDecompressBuffer, keyDecompressor);
        DataInputStream compressedIn = new DataInputStream(deflatFilter);
        deflatFilter.resetState();
        keyDecompressedData.reset();
        keyDecompressedData.write(compressedIn, currentKeyLength);
        keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength);
        currentKey.readFields(keyDataIn);
      } else {
View Full Code Here
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
      compressedKeyLen = in.readInt();
      if (decompress) {
        keyTempBuffer.reset();
        keyTempBuffer.write(in, compressedKeyLen);
        keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen);
        CompressionInputStream deflatFilter = codec.createInputStream(
            keyDecompressBuffer, keyDecompressor);
        DataInputStream compressedIn = new DataInputStream(deflatFilter);
        deflatFilter.resetState();
        keyDecompressedData.reset();
        keyDecompressedData.write(compressedIn, currentKeyLength);
        keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength);
        currentKey.readFields(keyDataIn);
      } else {
View Full Code Here
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
      compressedKeyLen = in.readInt();
      if (decompress) {
        keyTempBuffer.reset();
        keyTempBuffer.write(in, compressedKeyLen);
        keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen);
        CompressionInputStream deflatFilter = codec.createInputStream(
            keyDecompressBuffer, keyDecompressor);
        DataInputStream compressedIn = new DataInputStream(deflatFilter);
        deflatFilter.resetState();
        keyDecompressedData.reset();
        keyDecompressedData.write(compressedIn, currentKeyLength);
        keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength);
        currentKey.readFields(keyDataIn);
      } else {
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
    InputStream is = null;
    if (compressor != null) {
      // GZIPCodec fails w/ NPE if no configuration.
      if (compressor instanceof Configurable) ((Configurable)compressor).setConf(this.conf);
      Decompressor poolDecompressor = CodecPool.getDecompressor(compressor);
      CompressionInputStream cis =
        compressor.createInputStream(new ByteArrayInputStream(cellBlock, offset, length),
        poolDecompressor);
      ByteBufferOutputStream bbos = null;
      try {
        // TODO: This is ugly.  The buffer will be resized on us if we guess wrong.
View Full Code Here
212
213
214
215
216
217
218
219
220
221
222
   DefaultCodec codec = new DefaultCodec();
   codec.setConf(job);
   Path expectedFile = new Path(workDir, file + codec.getDefaultExtension());
   final FileInputStream istream = new FileInputStream(expectedFile.toString());
   CompressionInputStream cistream = codec.createInputStream(istream);
   LineReader reader = new LineReader(cistream);

   String output = "";
   Text out = new Text();
   while (reader.readLine(out) > 0) {
View Full Code Here
664
665
666
667
668
669
670
671
672
673
674
            {
              CompressionCodecFactory compressionCodecs =  new CompressionCodecFactory(job);
              final CompressionCodec codec = compressionCodecs.getCodec(file);
               if (codec != null) {
                  end = Long.MAX_VALUE;
                    CompressionInputStream stream = codec.createInputStream(fileIn);
                    this.xmlLoaderBPIS = new XMLLoaderBufferedPositionedInputStream(stream,start,end);
                  }
            }
           
            else
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.CompressionInputStream

  • com.dp.nebula.wormhole.plugins.reader.hivereader.HiveReader
  • com.hadoop.compression.snappy.CompressionTests
  • com.pinterest.secor.io.FileReaderWriterTest
  • hivemall.utils.hadoop.HadoopUtils
  • org.apache.hadoop.hbase.ipc.IPCUtil
  • org.apache.hadoop.hive.ql.io.RCFile$Reader
  • org.apache.hadoop.io.compress.lz4.TestLz4CompressorDecompressor
  • org.apache.hadoop.io.compress.snappy.TestSnappyCodec
  • org.apache.hadoop.io.compress.snappy.TestSnappyCompressorDecompressor
  • org.apache.hadoop.mapred.gridmix.CompressionEmulationUtil
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.