Package org.apache.hadoop.io.SequenceFile

Examples of org.apache.hadoop.io.SequenceFile.CompressionType

481
482
483
484
485
486
487
488
489
490
491
      if (rwonly && (!create || merge || fast)) {
        System.err.println(usage);
        System.exit(-1);
      }

      CompressionType compressionType =
        CompressionType.valueOf(compressType);
      CompressionCodec codec = (CompressionCodec)ReflectionUtils.newInstance(
                                                                             conf.getClassByName(compressionCodec),
                                                                             conf);
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
                                      throws IOException {

    Path file = new Path(job.getOutputPath(), name);
    FileSystem fs = file.getFileSystem(job);
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(job)) {
      // find the kind of compression to do
      compressionType = SequenceFile.getCompressionType(job);

      // find the right codec
View Full Code Here
392
393
394
395
396
397
398
399
400
401
402
        if (rwonly && (!create || merge || fast)) {
          System.err.println(usage);
          System.exit(-1);
        }

        CompressionType compressionType =
          CompressionType.valueOf(compressType);
        CompressionCodec codec = (CompressionCodec)ReflectionUtils.newInstance(
                                    conf.getClassByName(compressionCodec),
                                    conf);
View Full Code Here
498
499
500
501
502
503
504
505
506
507
508
                                                         Path file, Class<?> keyClass,
                                                         Class<?> valClass,
                                                         boolean isCompressed)
    throws IOException {
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    Class codecClass = null;
    if (isCompressed) {
      compressionType = SequenceFileOutputFormat.getOutputCompressionType(jc);
      codecClass = SequenceFileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
      codec = (CompressionCodec)
View Full Code Here
954
955
956
957
958
959
960
961
962
963
964
   * @return output stream over the created sequencefile
   */
  public static SequenceFile.Writer createSequenceWriter(JobConf jc, FileSystem fs, Path file,
      Class<?> keyClass, Class<?> valClass, boolean isCompressed) throws IOException {
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    Class codecClass = null;
    if (isCompressed) {
      compressionType = SequenceFileOutputFormat.getOutputCompressionType(jc);
      codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
      codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
View Full Code Here
226
227
228
229
230
231
232
233
234
235
236
              .forName(codecStr);
          FileOutputFormat.setOutputCompressorClass(jc_output, codec);
        }
        String type = conf.getCompressType();
        if (type != null && !type.trim().equals("")) {
          CompressionType style = CompressionType.valueOf(type);
          SequenceFileOutputFormat.setOutputCompressionType(jc, style);
        }
      }
      return getRecordWriter(jc_output, hiveOutputFormat, outputClass,
          isCompressed, tableInfo.getProperties(), outPath);
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
    // get the path of the temporary output file
    Path file = FileOutputFormat.getTaskOutputPath(job, name);
   
    FileSystem fs = file.getFileSystem(job);
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(job)) {
      // find the kind of compression to do
      compressionType = getOutputCompressionType(job);

      // find the right codec
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
        protected SequenceFile.Writer getSequenceWriter(TaskAttemptContext context)
                throws IOException {
              Configuration conf = context.getConfiguration();

              CompressionCodec codec = null;
              CompressionType compressionType = CompressionType.NONE;
              if (getCompressOutput(context)) {
                // find the kind of compression to do
                compressionType = getOutputCompressionType(context);
                // find the right codec
                Class<?> codecClass = getOutputCompressorClass(context,
View Full Code Here
36
37
38
39
40
41
42
43
  private CompressionOutputStream outStream;

  @Override
  public void open(String filePath, FlumeFormatter fmt) throws IOException {
    DefaultCodec defCodec = new DefaultCodec();
    CompressionType cType = CompressionType.BLOCK;
    open(filePath, defCodec, cType, fmt);
  }
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
    final SequenceFile.Writer[] outs = new SequenceFile.Writer[partitions];
    try {
      Reporter reporter = getReporter(umbilical, getProgress());
      FileSystem localFs = FileSystem.getNamed("local", job);
      CompressionCodec codec = null;
      CompressionType compressionType = CompressionType.NONE;
      if (job.getCompressMapOutput()) {
        // find the kind of compression to do, defaulting to record
        compressionType = SequenceFile.getCompressionType(job);

        // find the right codec
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.SequenceFile.CompressionType

  • com.asakusafw.bulkloader.extractor.DfsFileImport
  • com.asakusafw.bulkloader.extractor.DfsFileImportTest
  • com.backtype.hadoop.pail.SequenceFileFormat
  • com.linkedin.camus.etl.kafka.common.SequenceFileRecordWriterProvider
  • com.linkedin.json.JsonSequenceFileOutputFormat
  • com.mozilla.pig.storage.SeqFileMultiStorage$MultiStorageSequenceOutputFormat
  • org.apache.accumulo.server.master.LogSort$LoggerMapFileOutputFormat
  • org.apache.avro.mapreduce.AvroSequenceFileOutputFormat
  • org.apache.flume.sink.hdfs.HDFSCompressedDataStream
  • org.apache.hadoop.hive.ql.exec.FileSinkOperator
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.