Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.GzipCodec$GzipOutputStream$ResetableGZIPOutputStream


          getTableName());

      reader = SeqFileReader.getSeqFileReader(getDataFilePath().toString());

      if (codec == null) {
        codec = new GzipCodec();
      }
      assertTrue("Block compressed", reader.isBlockCompressed());
      assertEquals(codec.getClass(), reader.getCompressionCodec().getClass());

      // here we can actually instantiate (k, v) pairs.
View Full Code Here


      conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
    }
    FileSystem fs = FileSystem.get(conf);

    if (codec == null) {
      codec = new GzipCodec();
    }
    Path p = new Path(getDataFilePath().toString()
        + codec.getDefaultExtension());
    InputStream is = codec.createInputStream(fs.open(p));
    BufferedReader r = new BufferedReader(new InputStreamReader(is));
View Full Code Here

           
            // Use the codec according to the test case
            if (type.equals("bz2"))
               codec = new BZip2Codec();
            else if (type.equals("gz")) {
               codec = new GzipCodec();
               ((GzipCodec)codec).setConf(new Configuration());
            }

            CompressionInputStream createInputStream = codec
                  .createInputStream(new FileInputStream(file));
View Full Code Here

  private static CompressionCodec getCodec(Configuration conf,
      SqoopOptions options) throws IOException {
    if (options.shouldUseCompression()) {
      if (options.getCompressionCodec() == null) {
        return new GzipCodec();
      } else {
        return CodecMap.getCodec(options.getCompressionCodec(), conf);
      }
    }
    return null;
View Full Code Here

          getTableName());

      reader = SeqFileReader.getSeqFileReader(getDataFilePath().toString());

      if (codec == null) {
        codec = new GzipCodec();
      }
      assertTrue("Block compressed", reader.isBlockCompressed());
      assertEquals(codec.getClass(), reader.getCompressionCodec().getClass());

      // here we can actually instantiate (k, v) pairs.
View Full Code Here

      conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
    }
    FileSystem fs = FileSystem.get(conf);

    if (codec == null) {
      codec = new GzipCodec();
      ReflectionUtils.setConf(codec, getConf());
    }
    Path p = new Path(getDataFilePath().toString()
        + codec.getDefaultExtension());
    InputStream is = codec.createInputStream(fs.open(p));
View Full Code Here

    }
  }

  public void testNonSplittingGzipFile() throws IOException {
    SplittingOutputStream os  = new SplittingOutputStream(getConf(),
        getWritePath(), "nonsplit-", 0, new GzipCodec());
    SplittableBufferedWriter w = new SplittableBufferedWriter(os, true);
    try {
      w.allowSplit();
      w.write("This is a string!");
      w.newLine();
View Full Code Here

    }
  }

  public void testSplittingGzipFile() throws IOException {
    SplittingOutputStream os = new SplittingOutputStream(getConf(),
        getWritePath(), "splitz-", 3, new GzipCodec());
    SplittableBufferedWriter w = new SplittableBufferedWriter(os, true);
    try {
      w.write("This is a string!");
      w.newLine();
      w.write("This is another string!");
View Full Code Here

  public void testIFileWriterWithCodec() throws Exception {
    Configuration conf = new Configuration();
    FileSystem localFs = FileSystem.getLocal(conf);
    FileSystem rfs = ((LocalFileSystem)localFs).getRaw();
    Path path = new Path(new Path("build/test.ifile"), "data");
    DefaultCodec codec = new GzipCodec();
    codec.setConf(conf);
    IFile.Writer<Text, Text> writer =
      new IFile.Writer<Text, Text>(conf, rfs, path, Text.class, Text.class,
                                   codec, null);
    writer.close();
  }
View Full Code Here

  public void testIFileReaderWithCodec() throws Exception {
    Configuration conf = new Configuration();
    FileSystem localFs = FileSystem.getLocal(conf);
    FileSystem rfs = ((LocalFileSystem)localFs).getRaw();
    Path path = new Path(new Path("build/test.ifile"), "data");
    DefaultCodec codec = new GzipCodec();
    codec.setConf(conf);
    IFile.Writer<Text, Text> writer =
        new IFile.Writer<Text, Text>(conf, rfs, path, Text.class, Text.class,
                                     codec, null);
    writer.close();
    IFile.Reader<Text, Text> reader =
View Full Code Here

TOP

Related Classes of org.apache.hadoop.io.compress.GzipCodec$GzipOutputStream$ResetableGZIPOutputStream

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.