Examples of GzipCodec

Examples of org.apache.hadoop.io.compress.GzipCodec

609
610
611
612
613
614
615
616
617
618
619
620
        @Override
        public void writeTestData(File file, int recordCounts, int columnCount,
                String colSeparator) throws IOException {

            // write random test data
            GzipCodec gzipCodec = new GzipCodec();
            CompressionOutputStream out = gzipCodec
                    .createOutputStream(new FileOutputStream(file));
            BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(
                    out));

            try {
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

131
132
133
134
135
136
137
138
139
140
141
            // Use the codec according to the test case
            if (type.equals("bz2")) {
               codec = new BZip2Codec();
            } else if (type.equals("gz")) {
               codec = new GzipCodec();
            }
            if(codec instanceof Configurable) {
                ((Configurable)codec).setConf(new Configuration());
            }
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

49
50
51
52
53
54
55
 
  @Test
  public void testBuildCellBlock() throws IOException {
    doBuildCellBlockUndoCellBlock(new KeyValueCodec(), null);
    doBuildCellBlockUndoCellBlock(new KeyValueCodec(), new DefaultCodec());
    doBuildCellBlockUndoCellBlock(new KeyValueCodec(), new GzipCodec());
  }
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

63
64
65
66
67
68
69
70
71
72
73
  private static InputStream createStream(FSDataInputStream in, JobConf job)
    throws IOException{
    InputStream finalStream = in ;
    boolean gzipped = StreamInputFormat.isGzippedInput(job);
    if ( gzipped ) {
      GzipCodec codec = new GzipCodec();
      codec.setConf(job);
      finalStream = codec.createInputStream(in);
    }
    return finalStream;
  }
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

84
85
86
87
88
89
90
91
92
93
  public void testCompressedFile() throws Exception {
    String file = testFile.getCanonicalPath();
    HDFSCompressedDataStream stream = new HDFSCompressedDataStream();
    context.put("hdfs.useRawLocalFileSystem", "true");
    stream.configure(context);
    stream.open(file, new GzipCodec(), CompressionType.RECORD);
    stream.append(event);
    stream.sync();
    Assert.assertTrue(testFile.length() > 0);
  }
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
      LOG.warn("Config property "
          + FlumeConfiguration.COLLECTOR_DFS_COMPRESS_GZIP
          + " is deprecated, please use "
          + FlumeConfiguration.COLLECTOR_DFS_COMPRESS_CODEC
          + " set to GzipCodec instead");
      CompressionCodec gzipC = new GzipCodec();
      Compressor gzCmp = gzipC.createCompressor();
      dstPath = new Path(path + gzipC.getDefaultExtension());
      hdfs = dstPath.getFileSystem(conf);
      writer = hdfs.create(dstPath);
      writer = gzipC.createOutputStream(writer, gzCmp);
      LOG.info("Creating HDFS gzip compressed file: " + dstPath.toString());
      return;
    }

    String codecName = conf.getCollectorDfsCompressCodec();
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

219
220
221
222
223
224
225
226
227
228
229
   * Test using the gzip codec for reading
   */
  //@Test(timeout=10000)
  public void testGzip() throws IOException {
    JobConf job = new JobConf(defaultConf);
    CompressionCodec gzip = new GzipCodec();
    ReflectionUtils.setConf(gzip, job);
    localFs.delete(workDir, true);
    writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip,
              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

169
170
171
172
173
174
175
176
177
178
179
         }
        writer.close();
    } else if ("SequenceFile".equals(fileType)){
        CompressionCodec codec = null;
        if ("gz".equals(codecName))
          codec = new GzipCodec();
        else if (!"none".equals(codecName))
          throw new IOException("Codec not supported.");

        SequenceFile.Writer writer;
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

61
62
63
64
65
66
67
 
  @Test
  public void testBuildCellBlock() throws IOException {
    doBuildCellBlockUndoCellBlock(this.util, new KeyValueCodec(), null);
    doBuildCellBlockUndoCellBlock(this.util, new KeyValueCodec(), new DefaultCodec());
    doBuildCellBlockUndoCellBlock(this.util, new KeyValueCodec(), new GzipCodec());
  }
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

190
191
192
193
194
195
196
    }
    IPCUtil util = new IPCUtil(HBaseConfiguration.create());
    ((Log4JLogger)IPCUtil.LOG).getLogger().setLevel(Level.ALL);
    timerTests(util, count, size,  new KeyValueCodec(), null);
    timerTests(util, count, size,  new KeyValueCodec(), new DefaultCodec());
    timerTests(util, count, size,  new KeyValueCodec(), new GzipCodec());
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.