Package org.apache.avro.file

Examples of org.apache.avro.file.CodecFactory

108
109
110
111
112
113
114
115
116
117
118
   *   <li>Next use mapred.output.compression.codec if populated</li>
   *   <li>If not default to Deflate Codec</li>
   * </ul> 
   */
  static CodecFactory getCodecFactory(JobConf job) {
    CodecFactory factory = null;
   
    if (FileOutputFormat.getCompressOutput(job)) {
      int deflateLevel = job.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
      int xzLevel = job.getInt(XZ_LEVEL_KEY, DEFAULT_XZ_LEVEL);
      String codecName = job.get(AvroJob.OUTPUT_CODEC);
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
  *   <li>{@code org.apache.hadoop.io.compress.GZipCodec} will map to {@code deflate}</li>
  * </ul>
  */
  public static CodecFactory fromHadoopString(String hadoopCodecClass) {

    CodecFactory o = null;
    try {
      String avroCodec = HADOOP_AVRO_NAME_MAP.get(hadoopCodecClass);
      if (avroCodec != null) {
        o = CodecFactory.fromString(avroCodec);
      }
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
    AvroDatumConverter<Text, ?> keyConverter = factory.create(Text.class);
    AvroValue<TextStats> avroValue = new AvroValue<TextStats>(null);
    @SuppressWarnings("unchecked")
    AvroDatumConverter<AvroValue<TextStats>, ?> valueConverter
        = factory.create((Class<AvroValue<TextStats>>) avroValue.getClass());
    CodecFactory compressionCodec = CodecFactory.nullCodec();
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();

    // Use a writer to generate a Avro container file in memory.
    // Write two records: <'apple', TextStats('apple')> and <'banana', TextStats('banana')>.
    AvroKeyValueRecordWriter<Text, AvroValue<TextStats>> writer
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
    AvroDatumConverter<Text, ?> keyConverter = factory.create(Text.class);
    AvroValue<TextStats> avroValue = new AvroValue<TextStats>(null);
    @SuppressWarnings("unchecked")
    AvroDatumConverter<AvroValue<TextStats>, ?> valueConverter
        = factory.create((Class<AvroValue<TextStats>>) avroValue.getClass());
    CodecFactory compressionCodec = CodecFactory.nullCodec();
    FileOutputStream outputStream = new FileOutputStream(new File("target/temp.avro"));

    // Write a marker followed by each record: <'apple', TextStats('apple')> and <'banana', TextStats('banana')>.
    AvroKeyValueRecordWriter<Text, AvroValue<TextStats>> writer
        = new AvroKeyValueRecordWriter<Text, AvroValue<TextStats>>(keyConverter, valueConverter,
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
public class TestAvroKeyRecordWriter {
  @Test
  public void testWrite() throws IOException {
    Schema writerSchema = Schema.create(Schema.Type.INT);
    GenericData dataModel = new ReflectData();
    CodecFactory compressionCodec = CodecFactory.nullCodec();
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    TaskAttemptContext context = createMock(TaskAttemptContext.class);

    replay(context);
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
 
  @Test
  public void testSycnableWrite() throws IOException {
    Schema writerSchema = Schema.create(Schema.Type.INT);
    GenericData dataModel = new ReflectData();
    CodecFactory compressionCodec = CodecFactory.nullCodec();
    FileOutputStream outputStream = new FileOutputStream(new File("target/temp.avro"));
    TaskAttemptContext context = createMock(TaskAttemptContext.class);

    replay(context);
View Full Code Here
24
25
26
27
28
29
30
31
32
public class TestHadoopCodecFactory {
 
  @Test
  public void testHadoopCodecFactoryDeflate(){
    CodecFactory hadoopDeflateCodec = HadoopCodecFactory.fromHadoopString("org.apache.hadoop.io.compress.DeflateCodec");
    CodecFactory avroDeflateCodec = CodecFactory.fromString("deflate");
    assertTrue(hadoopDeflateCodec.getClass().equals(avroDeflateCodec.getClass()));
  }
View Full Code Here
31
32
33
34
35
36
37
38
39
    assertTrue(hadoopDeflateCodec.getClass().equals(avroDeflateCodec.getClass()));
  }
 
  @Test
  public void testHadoopCodecFactorySnappy(){
    CodecFactory hadoopSnappyCodec = HadoopCodecFactory.fromHadoopString("org.apache.hadoop.io.compress.SnappyCodec");
    CodecFactory avroSnappyCodec = CodecFactory.fromString("snappy");
    assertTrue(hadoopSnappyCodec.getClass().equals(avroSnappyCodec.getClass()));
  }
View Full Code Here
38
39
40
41
42
43
44
45
46
    assertTrue(hadoopSnappyCodec.getClass().equals(avroSnappyCodec.getClass()));
  }
 
  @Test
  public void testHadoopCodecFactoryBZip2(){
    CodecFactory hadoopSnappyCodec = HadoopCodecFactory.fromHadoopString("org.apache.hadoop.io.compress.BZip2Codec");
    CodecFactory avroSnappyCodec = CodecFactory.fromString("bzip2");
    assertTrue(hadoopSnappyCodec.getClass().equals(avroSnappyCodec.getClass()));
  }
View Full Code Here
45
46
47
48
49
50
51
52
53
    assertTrue(hadoopSnappyCodec.getClass().equals(avroSnappyCodec.getClass()));
  }
 
  @Test
  public void testHadoopCodecFactoryGZip(){
    CodecFactory hadoopSnappyCodec = HadoopCodecFactory.fromHadoopString("org.apache.hadoop.io.compress.GZipCodec");
    CodecFactory avroSnappyCodec = CodecFactory.fromString("deflate");
    assertTrue(hadoopSnappyCodec.getClass().equals(avroSnappyCodec.getClass()));
  }
View Full Code Here
TOP

Related Classes of org.apache.avro.file.CodecFactory

  • com.cloudera.flume.handlers.avro.AvroDataFileOutputFormat
  • com.datasalt.pangool.examples.avro.TestAvroTweetsJoin
  • com.datasalt.pangool.tuplemr.avro.AvroOutputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.output.TupleOutputFormat
  • com.linkedin.haivvreo.AvroContainerOutputFormat
  • org.apache.avro.hadoop.file.HadoopCodecFactory
  • org.apache.avro.hadoop.file.TestHadoopCodecFactory
  • org.apache.avro.mapred.AvroOutputFormat
  • org.apache.avro.mapred.TestAvroOutputFormat
  • org.apache.avro.mapreduce.lib.output.AvroOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.