Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.DefaultCodec

990
991
992
993
994
995
996
997
998
999
1000
          } catch (ClassNotFoundException cnfe) {
            throw new IllegalArgumentException("Unknown codec: " +
                                               codecClassname, cnfe);
          }
        } else {
          codec = new DefaultCodec();
        }
      }
     
      if (version > 1) {                          // if version > 1
        in.readFully(sync);                       // read sync bytes
View Full Code Here
178
179
180
181
182
183
184
185
186
187
188
        new RCFile.Writer(fs, conf, file, null,
            RCFile.createMetadata(new Text("apple"),
                new Text("block"),
                new Text("cat"),
                new Text("dog")),
            new DefaultCodec());
    BytesRefArrayWritable bytes = new BytesRefArrayWritable(record_1.length);
    for (int i = 0; i < record_1.length; i++) {
      BytesRefWritable cu = new BytesRefWritable(record_1[i], 0,
          record_1[i].length);
      bytes.set(i, cu);
View Full Code Here
383
384
385
386
387
388
389
390
391
392
393
                         byte[][] fieldsData, Configuration conf) throws IOException {
    fs.delete(file, true);

    conf.setInt(RCFile.COLUMN_NUMBER_CONF_STR, fieldsData.length);
    RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null,
        new DefaultCodec());

    BytesRefArrayWritable bytes = new BytesRefArrayWritable(fieldsData.length);
    for (int i = 0; i < fieldsData.length; i++) {
      BytesRefWritable cu;
      cu = new BytesRefWritable(fieldsData[i], 0, fieldsData[i].length);
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
      conf.setInt(RCFile.COLUMN_NUMBER_CONF_STR, schema.getColumnNum());
      boolean compress = meta.getOption("rcfile.compress") != null &&
          meta.getOption("rcfile.compress").equalsIgnoreCase("true");
      if (compress) {
        writer = new RCFile.Writer(fs, conf, path, null, new DefaultCodec());
      } else {
        writer = new RCFile.Writer(fs, conf, path, null, null);
      }

      if (enabledStats) {
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
      (short) conf.getInt("hbase.regionserver.hlog.replication",
        fs.getDefaultReplication()),
      conf.getLong("hbase.regionserver.hlog.blocksize",
        fs.getDefaultBlockSize()),
      SequenceFile.CompressionType.NONE,
      new DefaultCodec(),
      null,
      new Metadata());

    // Get at the private FSDataOutputStream inside in SequenceFile so we can
    // call sync on it.  Make it accessible.  Stash it aside for call up in
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
    FileSystem fs = FileSystem.getLocal(conf);
    fs.delete(file, true);

    byte [][][] records = getRecords();
    RCFileOutputFormat.setColumnNumber(conf, 8);
    RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, new DefaultCodec());

    BytesRefArrayWritable bytes = writeBytesToFile(records[0], writer);
    BytesRefArrayWritable bytes2 = writeBytesToFile(records[1], writer);

    writer.close();
View Full Code Here
943
944
945
946
947
948
949
950
        this(value, null);
      }
      CompressionOption(CompressionType value, CompressionCodec codec) {
        this.value = value;
        this.codec = (CompressionType.NONE != value && null == codec)
          ? new DefaultCodec()
          : codec;
      }
View Full Code Here
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
          } catch (ClassNotFoundException cnfe) {
            throw new IllegalArgumentException("Unknown codec: " +
                                               codecClassname, cnfe);
          }
        } else {
          codec = new DefaultCodec();
          ((Configurable)codec).setConf(conf);
        }
      }
     
      this.metadata = new Metadata();
View Full Code Here
243
244
245
246
247
248
249
250
251
252
253
254
255
256
    }
  }

  @Test
  public void testDecompressedStream() throws Exception {
    DefaultCodec codec = new DefaultCodec();
    codec.setConf(fs.getConf());
    String name = "local/" + UUID.randomUUID() + codec.getDefaultExtension();
    OutputStream outputStream = codec.createOutputStream(fs.create(new Path(name)));
    byte[] content = name.getBytes();
    outputStream.write(content);
    outputStream.close();

    Resource resource = loader.getResource(name);
View Full Code Here
261
262
263
264
265
266
267
268
269
270
271
272
273
274
  }

  @Test
  public void testCompressedStream() throws Exception {

    DefaultCodec codec = new DefaultCodec();
    codec.setConf(fs.getConf());
    String name = "local/" + UUID.randomUUID() + codec.getDefaultExtension();
    OutputStream outputStream = codec.createOutputStream(fs.create(new Path(name)));
    byte[] content = name.getBytes();
    outputStream.write(content);
    outputStream.close();

    loader.setUseCodecs(false);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.DefaultCodec

  • com.alexholmes.hadooputils.combine.avro.AvroFileGenerator
  • com.alexholmes.hadooputils.combine.seqfile.mapred.CombineSequenceFileTest
  • com.alexholmes.hadooputils.combine.seqfile.mapreduce.CombineSequenceFileTest
  • com.alexholmes.hadooputils.combine.seqfile.SequenceFileGenerator
  • com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest
  • com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest
  • com.cloudera.flume.handlers.hdfs.TestDFSWrite
  • com.cloudera.flume.handlers.hdfs.TestEscapedCustomOutputDfs
  • com.m6d.filecrush.crush.CrushReducerParameterizedTest
  • com.m6d.filecrush.crush.integration.CrushMapReduceTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.