Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.GzipCodec$GzipInputStream

177
178
179
180
181
182
183
184
185
186
187
         }
        writer.close();
    } else if ("SequenceFile".equals(fileType)){
        CompressionCodec codec = null;
        if ("gz".equals(codecName))
          codec = new GzipCodec();
        else if (!"none".equals(codecName))
          throw new IOException("Codec not supported.");

        SequenceFile.Writer writer;
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
           
            // Use the codec according to the test case
            if (type.equals("bz2"))
               codec = new BZip2Codec();
            else if (type.equals("gz")) {
               codec = new GzipCodec();
               ((GzipCodec)codec).setConf(new Configuration());
            }

            CompressionInputStream createInputStream = codec
                  .createInputStream(new FileInputStream(file));
View Full Code Here
608
609
610
611
612
613
614
615
616
617
618
619
        @Override
        public void writeTestData(File file, int recordCounts, int columnCount,
                String colSeparator) throws IOException {

            // write random test data
            GzipCodec gzipCodec = new GzipCodec();
            CompressionOutputStream out = gzipCodec
                    .createOutputStream(new FileOutputStream(file));
            BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(
                    out));

            try {
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
         }
        writer.close();
    } else if ("SequenceFile".equals(fileType)){
        CompressionCodec codec = null;
        if ("gz".equals(codecName))
          codec = new GzipCodec();
        else if (!"none".equals(codecName))
          throw new IOException("Codec not supported.");

        SequenceFile.Writer writer;
View Full Code Here
37
38
39
40
41
42
43
44
45
46
47
48
  public void testIFileWriterWithCodec() throws Exception {
    Configuration conf = new Configuration();
    FileSystem localFs = FileSystem.getLocal(conf);
    FileSystem rfs = ((LocalFileSystem)localFs).getRaw();
    Path path = new Path(new Path("build/test.ifile"), "data");
    DefaultCodec codec = new GzipCodec();
    codec.setConf(conf);
    IFile.Writer<Text, Text> writer =
      new IFile.Writer<Text, Text>(conf, rfs, path, Text.class, Text.class,
                                   codec, null);
    writer.close();
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
  public void testIFileReaderWithCodec() throws Exception {
    Configuration conf = new Configuration();
    FileSystem localFs = FileSystem.getLocal(conf);
    FileSystem rfs = ((LocalFileSystem)localFs).getRaw();
    Path path = new Path(new Path("build/test.ifile"), "data");
    DefaultCodec codec = new GzipCodec();
    codec.setConf(conf);
    IFile.Reader<Text, Text> reader =
      new IFile.Reader<Text, Text>(conf, rfs, path, codec, null);
    reader.close();
  }
View Full Code Here
24
25
26
27
28
29
30
31
32
33
34
        if ( keyClass==null) keyClass=Text.class;
        if ( valClass==null) valClass=BytesWritable.class;
        if ( ct == null )
            ct = CompressionType.BLOCK;
        if ( codec == null )
            codec = new GzipCodec();
       
        Configuration conf = new Configuration();
        Path p = new Path(hdfsPath);
        FileSystem fs = FileSystem.get(conf);
        SequenceFile.Writer w = SequenceFile.createWriter(fs,conf,p,keyClass,valClass,ct,codec);
View Full Code Here
138
139
140
141
142
143
144
   * @throws InterruptedException
   */
  @Test
  public void testGZipCodec() throws IOException, InterruptedException {
    checkOutputFormat("syslog", new SyslogEntryFormat(), "GzipCodec",
        new GzipCodec());
  }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
    }

  @Test
  public void testSpillListCompressed()
    {
    GzipCodec codec = ReflectionUtils.newInstance( GzipCodec.class, new Configuration() );

    long time = System.currentTimeMillis();

    performListTest( 5, 50, codec, 0 );
    performListTest( 49, 50, codec, 0 );
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
         }
        writer.close();
    } else if ("SequenceFile".equals(fileType)){
        CompressionCodec codec = null;
        if ("gz".equals(codecName))
          codec = new GzipCodec();
        else if (!"none".equals(codecName))
          throw new IOException("Codec not supported.");

        SequenceFile.Writer writer;
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.GzipCodec$GzipInputStream

  • cascading.tuple.hadoop.SpillableTupleHadoopTest
  • com.cloudera.flume.handlers.hdfs.CustomDfsSink
  • com.cloudera.flume.handlers.hdfs.TestEscapedCustomOutputDfs
  • com.cloudera.sqoop.io.TestSplittableBufferedWriter
  • com.cloudera.sqoop.TestCompression
  • com.fb2pdf.hadoop.cluster.SequenceFilesFromDirectory$ChunkedWriter
  • com.pinterest.secor.common.FileRegistryTest
  • com.pinterest.secor.io.FileReaderWriterTest
  • org.apache.flume.sink.hdfs.TestUseRawLocalFileSystem
  • org.apache.hadoop.hbase.io.hfile.TestHFilePerformance
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.