Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.DefaultCodec

1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
          } catch (ClassNotFoundException cnfe) {
            throw new IllegalArgumentException("Unknown codec: " +
                                               codecClassname, cnfe);
          }
        } else {
          codec = new DefaultCodec();
          ((Configurable)codec).setConf(conf);
        }
      }
     
      if (version > 1) {                          // if version > 1
View Full Code Here
42
43
44
45
46
47
48
49
  public TestSequenceFile(String name) { super(name); }

  /** Unit tests for SequenceFile. */
  public void testZlibSequenceFile() throws Exception {
    LOG.info("Testing SequenceFile with DefaultCodec");
    compressedSeqFileTest(new DefaultCodec());
    LOG.info("Successfully tested SequenceFile with DefaultCodec");
  }
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
  }
 
  public void testValueIteratorWithCompression() throws Exception {
    Path tmpDir = new Path("build/test/test.reduce.task.compression");
    Configuration conf = new Configuration();
    DefaultCodec codec = new DefaultCodec();
    codec.setConf(conf);
    for (Pair[] testCase: testCases) {
      runValueIterator(tmpDir, testCase, conf, codec);
    }
  }
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
        "5.3".getBytes("UTF-8"), "hive and hadoop".getBytes("UTF-8"),
        new byte[0], new byte[0] };

    RCFileOutputFormat.setColumnNumber(conf, expectedFieldsData.length);
    RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null,
        new DefaultCodec());
    BytesRefArrayWritable bytes = new BytesRefArrayWritable(record_1.length);
    for (int i = 0; i < record_1.length; i++) {
      BytesRefWritable cu = new BytesRefWritable(record_1[i], 0,
          record_1[i].length);
      bytes.set(i, cu);
View Full Code Here
251
252
253
254
255
256
257
258
259
260
261
      byte[][] fieldsData) throws IOException, SerDeException {
    fs.delete(file, true);

    RCFileOutputFormat.setColumnNumber(conf, fieldsData.length);
    RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null,
        new DefaultCodec());

    BytesRefArrayWritable bytes = new BytesRefArrayWritable(fieldsData.length);
    for (int i = 0; i < fieldsData.length; i++) {
      BytesRefWritable cu = null;
      cu = new BytesRefWritable(fieldsData[i], 0, fieldsData[i].length);
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
      PerformTestRCFileAndSeqFile testcase = new PerformTestRCFileAndSeqFile(
          isLocalFS, file);

      // change these parameters
      boolean checkCorrect = true;
      CompressionCodec codec = new DefaultCodec();
      testcase.columnMaxSize = 30;

      // testcase.testWithColumnNumber(count, 2, checkCorrect, codec);
      // testcase.testWithColumnNumber(count, 10, checkCorrect, codec);
      // testcase.testWithColumnNumber(count, 25, checkCorrect, codec);
View Full Code Here
227
228
229
230
231
232
233
234
235
236
237
    Configuration conf = createConfiguration(outputContext, Text.class, Text.class, shouldCompress,
        -1, HashPartitioner.class);
    CompressionCodec codec = null;
    if (shouldCompress) {
      codec = new DefaultCodec();
      ((Configurable) codec).setConf(conf);
    }

    int numRecordsWritten = 0;
View Full Code Here
399
400
401
402
403
404
405
406
407
408
409
    Configuration conf = createConfiguration(outputContext, IntWritable.class, LongWritable.class,
        shouldCompress, -1);
    CompressionCodec codec = null;
    if (shouldCompress) {
      codec = new DefaultCodec();
      ((Configurable) codec).setConf(conf);
    }

    int numOutputs = numPartitions;
    long availableMemory = 2048;
View Full Code Here
282
283
284
285
286
287
288
                 Class keyClass, Class valClass, CompressionType compressionType)
    throws IOException {
    return createWriter(fs, conf, name, keyClass, valClass,
            fs.getConf().getInt("io.file.buffer.size", 4096),
            fs.getDefaultReplication(), fs.getDefaultBlockSize(),
            compressionType, new DefaultCodec(), null, new Metadata());
  }
View Full Code Here
304
305
306
307
308
309
310
                 Class keyClass, Class valClass, CompressionType compressionType,
                 Progressable progress) throws IOException {
    return createWriter(fs, conf, name, keyClass, valClass,
            fs.getConf().getInt("io.file.buffer.size", 4096),
            fs.getDefaultReplication(), fs.getDefaultBlockSize(),
            compressionType, new DefaultCodec(), progress, new Metadata());
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.DefaultCodec

  • com.alexholmes.hadooputils.combine.avro.AvroFileGenerator
  • com.alexholmes.hadooputils.combine.seqfile.mapred.CombineSequenceFileTest
  • com.alexholmes.hadooputils.combine.seqfile.mapreduce.CombineSequenceFileTest
  • com.alexholmes.hadooputils.combine.seqfile.SequenceFileGenerator
  • com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest
  • com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest
  • com.cloudera.flume.handlers.hdfs.TestDFSWrite
  • com.cloudera.flume.handlers.hdfs.TestEscapedCustomOutputDfs
  • com.m6d.filecrush.crush.CrushReducerParameterizedTest
  • com.m6d.filecrush.crush.integration.CrushMapReduceTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.