Examples of GzipCodec

Examples of org.apache.hadoop.io.compress.GzipCodec

73
74
75
76
77
78
79
80
81
82
  public void testCompressedFile() throws Exception {
    String file = testFile.getCanonicalPath();
    HDFSCompressedDataStream stream = new HDFSCompressedDataStream();
    context.put("hdfs.useRawLocalFileSystem", "true");
    stream.configure(context);
    stream.open(file, new GzipCodec(), CompressionType.RECORD);
    stream.append(event);
    stream.sync();
    Assert.assertTrue(testFile.length() > 0);
  }
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

179
180
181
182
183
184
185
186
187
188
189
         }
        writer.close();
    } else if ("SequenceFile".equals(fileType)){
        CompressionCodec codec = null;
        if ("gz".equals(codecName))
          codec = new GzipCodec();
        else if (!"none".equals(codecName))
          throw new IOException("Codec not supported.");

        SequenceFile.Writer writer;
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

208
209
210
211
212
213
214
215
216
217
218
   * Test using the gzip codec for reading
   */
  @Test(timeout=10000)
  public void testGzip() throws IOException {
    JobConf job = new JobConf(defaultConf);
    CompressionCodec gzip = new GzipCodec();
    ReflectionUtils.setConf(gzip, job);
    localFs.delete(workDir, true);
    writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip,
              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

224
225
226
227
228
229
230
231
232
233
234
   * Test using the gzip codec for reading
   */
  @Test(timeout=10000)
  public void testGzip() throws IOException, InterruptedException {
    Configuration conf = new Configuration(defaultConf);
    CompressionCodec gzip = new GzipCodec();
    ReflectionUtils.setConf(gzip, conf);
    localFs.delete(workDir, true);
    writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip,
              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

128
129
130
131
132
133
134
135
                .thenReturn(writer);

        Mockito.when(writer.getLength()).thenReturn(123L);

        FileReaderWriter createdWriter = mRegistry.getOrCreateWriter(
                mLogFilePathGz, new GzipCodec());
        assertTrue(createdWriter == writer);
    }
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

135
136
137
138
139
140
141
142
143
144
145
    }

    public void testGetOrCreateWriterCompressed() throws Exception {
        createCompressedWriter();

        mRegistry.getOrCreateWriter(mLogFilePathGz, new GzipCodec());

        // Verify that the method has been called exactly once (the default).
        PowerMockito.verifyStatic();
        FileUtil.delete(PATH_GZ);
        PowerMockito.verifyStatic();
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
                "com.pinterest.secor.io.impl.DelimitedTextFileReaderWriter");
        mConfig = new SecorConfig(properties);
    }

    private void mockGzipFileReaderWriter() throws Exception {
        GzipCodec codec = PowerMockito.mock(GzipCodec.class);
        PowerMockito.whenNew(GzipCodec.class).withNoArguments()
                .thenReturn(codec);

        FileInputStream fileInputStream = Mockito.mock(FileInputStream.class);
        FileOutputStream fileOutputStream = Mockito.mock(FileOutputStream.class);

        PowerMockito.whenNew(FileInputStream.class).withAnyArguments()
                .thenReturn(fileInputStream);

        PowerMockito.whenNew(FileOutputStream.class).withAnyArguments()
                .thenReturn(fileOutputStream);

        CompressionInputStream inputStream = Mockito
                .mock(CompressionInputStream.class);
        CompressionOutputStream outputStream = Mockito
                .mock(CompressionOutputStream.class);
        Mockito.when(codec.createInputStream(Mockito.any(InputStream.class)))
                .thenReturn(inputStream);

        Mockito.when(codec.createOutputStream(Mockito.any(OutputStream.class)))
                .thenReturn(outputStream);
    }
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

188
189
190
191
192
193
194
195
196
197
198
        assert writer.getLength() == 123L;

        mockSequenceFileReaderWriter(true);

        ReflectionUtil.createFileReaderWriter(mConfig.getFileReaderWriter(),
                mLogFilePathGz, new GzipCodec(), FileReaderWriter.Type.Writer);

        // Verify that the method has been called exactly once (the default).
        PowerMockito.verifyStatic();
        FileSystem.get(Mockito.any(Configuration.class));
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

202
203
204
205
206
207
208
209
    public void testGzipFileWriter() throws Exception {
        setupGzipFileReaderConfig();
        mockGzipFileReaderWriter();
        FileReaderWriter writer = (FileReaderWriter) ReflectionUtil
                .createFileReaderWriter(mConfig.getFileReaderWriter(),
                        mLogFilePathGz, new GzipCodec(), FileReaderWriter.Type.Writer);
        assert writer.getLength() == 0L;
    }
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec

210
211
212
213
214
215
216
    public void testGzipFileReader() throws Exception {
        setupGzipFileReaderConfig();
        mockGzipFileReaderWriter();
        ReflectionUtil.createFileReaderWriter(mConfig.getFileReaderWriter(),
                mLogFilePathGz, new GzipCodec(), FileReaderWriter.Type.Reader);
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.