Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.GzipCodec$GzipOutputStream$ResetableGZIPOutputStream


  public void testCompressedFile() throws Exception {
    String file = testFile.getCanonicalPath();
    HDFSCompressedDataStream stream = new HDFSCompressedDataStream();
    context.put("hdfs.useRawLocalFileSystem", "true");
    stream.configure(context);
    stream.open(file, new GzipCodec(), CompressionType.RECORD);
    stream.append(event);
    stream.sync();
    Assert.assertTrue(testFile.length() > 0);
  }
View Full Code Here


         }
        writer.close();
    } else if ("SequenceFile".equals(fileType)){
        CompressionCodec codec = null;
        if ("gz".equals(codecName))
          codec = new GzipCodec();
        else if (!"none".equals(codecName))
          throw new IOException("Codec not supported.");

        SequenceFile.Writer writer;
View Full Code Here

   * Test using the gzip codec for reading
   */
  @Test(timeout=10000)
  public void testGzip() throws IOException {
    JobConf job = new JobConf(defaultConf);
    CompressionCodec gzip = new GzipCodec();
    ReflectionUtils.setConf(gzip, job);
    localFs.delete(workDir, true);
    writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip,
              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,
View Full Code Here

   * Test using the gzip codec for reading
   */
  @Test(timeout=10000)
  public void testGzip() throws IOException, InterruptedException {
    Configuration conf = new Configuration(defaultConf);
    CompressionCodec gzip = new GzipCodec();
    ReflectionUtils.setConf(gzip, conf);
    localFs.delete(workDir, true);
    writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip,
              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,
View Full Code Here

                .thenReturn(writer);

        Mockito.when(writer.getLength()).thenReturn(123L);

        FileReaderWriter createdWriter = mRegistry.getOrCreateWriter(
                mLogFilePathGz, new GzipCodec());
        assertTrue(createdWriter == writer);
    }
View Full Code Here

    }

    public void testGetOrCreateWriterCompressed() throws Exception {
        createCompressedWriter();

        mRegistry.getOrCreateWriter(mLogFilePathGz, new GzipCodec());

        // Verify that the method has been called exactly once (the default).
        PowerMockito.verifyStatic();
        FileUtil.delete(PATH_GZ);
        PowerMockito.verifyStatic();
View Full Code Here

                "com.pinterest.secor.io.impl.DelimitedTextFileReaderWriter");
        mConfig = new SecorConfig(properties);
    }

    private void mockGzipFileReaderWriter() throws Exception {
        GzipCodec codec = PowerMockito.mock(GzipCodec.class);
        PowerMockito.whenNew(GzipCodec.class).withNoArguments()
                .thenReturn(codec);

        FileInputStream fileInputStream = Mockito.mock(FileInputStream.class);
        FileOutputStream fileOutputStream = Mockito.mock(FileOutputStream.class);

        PowerMockito.whenNew(FileInputStream.class).withAnyArguments()
                .thenReturn(fileInputStream);

        PowerMockito.whenNew(FileOutputStream.class).withAnyArguments()
                .thenReturn(fileOutputStream);

        CompressionInputStream inputStream = Mockito
                .mock(CompressionInputStream.class);
        CompressionOutputStream outputStream = Mockito
                .mock(CompressionOutputStream.class);
        Mockito.when(codec.createInputStream(Mockito.any(InputStream.class)))
                .thenReturn(inputStream);

        Mockito.when(codec.createOutputStream(Mockito.any(OutputStream.class)))
                .thenReturn(outputStream);
    }
View Full Code Here

        assert writer.getLength() == 123L;

        mockSequenceFileReaderWriter(true);

        ReflectionUtil.createFileReaderWriter(mConfig.getFileReaderWriter(),
                mLogFilePathGz, new GzipCodec(), FileReaderWriter.Type.Writer);

        // Verify that the method has been called exactly once (the default).
        PowerMockito.verifyStatic();
        FileSystem.get(Mockito.any(Configuration.class));
View Full Code Here

    public void testGzipFileWriter() throws Exception {
        setupGzipFileReaderConfig();
        mockGzipFileReaderWriter();
        FileReaderWriter writer = (FileReaderWriter) ReflectionUtil
                .createFileReaderWriter(mConfig.getFileReaderWriter(),
                        mLogFilePathGz, new GzipCodec(), FileReaderWriter.Type.Writer);
        assert writer.getLength() == 0L;
    }
View Full Code Here

    public void testGzipFileReader() throws Exception {
        setupGzipFileReaderConfig();
        mockGzipFileReaderWriter();
        ReflectionUtil.createFileReaderWriter(mConfig.getFileReaderWriter(),
                mLogFilePathGz, new GzipCodec(), FileReaderWriter.Type.Reader);
    }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.io.compress.GzipCodec$GzipOutputStream$ResetableGZIPOutputStream

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.