Package org.apache.avro.file

Examples of org.apache.avro.file.CodecFactory

113
114
115
116
117
118
119
120
121
122
123
          if (args.length > 1) {
            compressionLevel = Integer.parseInt(args[1]);
          }
        }

        CodecFactory codecFactory = null;
        try {
          if ((codecName != null) && codecName.equals("deflate") && (compressionLevel != -1)) {
            codecFactory = CodecFactory.deflateCodec(compressionLevel);
          } else if (codecName != null) {
            codecFactory = CodecFactory.fromString(codecName);
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
    new File(where).delete();
   
    DataFileWriter<Record> writer =
        new DataFileWriter<Record>(new ReflectDatumWriter<Record>());
   
    CodecFactory factory = CodecFactory.deflateCodec(1);
    org.apache.avro.Schema schema = AvroTweetsJoin.getAvroTweetSchema();
   
    writer.setCodec(factory);
    int SYNC_SIZE = 16;
    int DEFAULT_SYNC_INTERVAL = 1000*SYNC_SIZE;
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
    Configuration conf = context.getConfiguration();
    if(conf.getBoolean("mapred.output.compress", false)) {
      String codec = conf.get("mapred.output.compression");
      int level = conf.getInt(AvroOutputFormat.DEFLATE_LEVEL_KEY,
          AvroOutputFormat.DEFAULT_DEFLATE_LEVEL);
      CodecFactory factory = codec.equals(DEFLATE_CODEC) ? CodecFactory
          .deflateCodec(level) : CodecFactory.fromString(codec);
      writer.setCodec(factory);
    }
    writer.setSyncInterval(conf.getInt(AvroOutputFormat.SYNC_INTERVAL_KEY,
        DEFAULT_SYNC_INTERVAL));
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
  }
 
  static <T> void configureDataFileWriter(DataFileWriter<T> writer,
      JobConf job) throws UnsupportedEncodingException {
   
    CodecFactory factory = getCodecFactory(job);
   
    if (factory != null) {
      writer.setCodec(factory); 
    }
   
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
   *   <li>Next use mapred.output.compression.codec if populated</li>
   *   <li>If not default to Deflate Codec</li>
   * </ul> 
   */
  static CodecFactory getCodecFactory(JobConf job) {
    CodecFactory factory = null;
   
    if (FileOutputFormat.getCompressOutput(job)) {
      int level = job.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
      String codecName = job.get(AvroJob.OUTPUT_CODEC);
     
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
public class TestAvroKeyRecordWriter {
  @Test
  public void testWrite() throws IOException {
    Schema writerSchema = Schema.create(Schema.Type.INT);
    CodecFactory compressionCodec = CodecFactory.nullCodec();
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    TaskAttemptContext context = createMock(TaskAttemptContext.class);

    replay(context);
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
  }
 
  static <T> void configureDataFileWriter(DataFileWriter<T> writer,
      JobConf job) throws UnsupportedEncodingException {
   
    CodecFactory factory = getCodecFactory(job);
   
    if (factory != null) {
      writer.setCodec(factory); 
    }
   
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
 
  static <T> void configureDataFileWriter(DataFileWriter<T> writer,
      TaskAttemptContext job,String codecName,int deflateLevel) throws UnsupportedEncodingException {
    Configuration conf = job.getConfiguration();
    if (FileOutputFormat.getCompressOutput(job)) {
      CodecFactory factory = codecName.equals(DEFLATE_CODEC)
        ? CodecFactory.deflateCodec(deflateLevel)
        : CodecFactory.fromString(codecName);
      writer.setCodec(factory);
    }
   
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
      } else {
        codec = "null";
      }
      int level = conf.getInt(AvroOutputFormat.DEFLATE_LEVEL_KEY,
          AvroOutputFormat.DEFAULT_DEFLATE_LEVEL);
      CodecFactory factory = codec.equals(DEFLATE_CODEC) ? CodecFactory
          .deflateCodec(level) : CodecFactory.fromString(codec);
      writer.setCodec(factory);
    }
    writer.setSyncInterval(conf.getInt(AvroOutputFormat.SYNC_INTERVAL_KEY,
        DEFAULT_SYNC_INTERVAL));
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
      TaskAttemptContext job) throws UnsupportedEncodingException {
    Configuration conf = job.getConfiguration();
    if (FileOutputFormat.getCompressOutput(job)) {
      int level = conf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
      String codecName = conf.get(AvroJob.OUTPUT_CODEC, DEFLATE_CODEC);
      CodecFactory factory = codecName.equals(DEFLATE_CODEC)
        ? CodecFactory.deflateCodec(level)
        : CodecFactory.fromString(codecName);
      writer.setCodec(factory);
    }
   
View Full Code Here
TOP

Related Classes of org.apache.avro.file.CodecFactory

  • com.cloudera.flume.handlers.avro.AvroDataFileOutputFormat
  • com.datasalt.pangool.examples.avro.TestAvroTweetsJoin
  • com.datasalt.pangool.tuplemr.avro.AvroOutputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.output.TupleOutputFormat
  • com.linkedin.haivvreo.AvroContainerOutputFormat
  • org.apache.avro.hadoop.file.HadoopCodecFactory
  • org.apache.avro.hadoop.file.TestHadoopCodecFactory
  • org.apache.avro.mapred.AvroOutputFormat
  • org.apache.avro.mapred.TestAvroOutputFormat
  • org.apache.avro.mapreduce.lib.output.AvroOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.