Examples of createOutputStream()


Examples of org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()

    fs.mkdirs(tablePath);
    OutputStream os = fs.create(filePath);
    if (gzip) {
      CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
      CompressionCodec codec = ccf.getCodec(filePath);
      os = codec.createOutputStream(os);
    }
    BufferedWriter w = new BufferedWriter(new OutputStreamWriter(os));
    for (int i = 0; i < numRecords; i++) {
      w.write(getRecordLine(startId + i, extraCols));
    }
View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()

    // open input stream
    InputStream in = srcFs.open(srcPath);

    // Create compression stream
    OutputStream out = dstFs.create(dstPath);
    out = codec.createOutputStream(out);

    IOUtils.copyBytes(in, out, conf, true);

    return 0;
  }
View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()

      boolean isCompressed) throws IOException {
    if (isCompressed) {
      Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jc,
          DefaultCodec.class);
      CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
      return codec.createOutputStream(out);
    } else {
      return (out);
    }
  }
View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()

      // Test a simple codec. Extension based detection. We use
      // Bzip2 cause its non-native.
      CompressionCodec codec = ReflectionUtils.newInstance(BZip2Codec.class, conf);
      String extension = codec.getDefaultExtension();
      Path p = new Path(root, "file." + extension);
      OutputStream fout = new DataOutputStream(codec.createOutputStream(
          fs.create(p, true)));
      byte[] writebytes = "foo".getBytes();
      fout.write(writebytes);
      fout.close();
      out = new ByteArrayOutputStream();
View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()

      // build the filename including the extension
      Path file = FileOutputFormat.getTaskOutputPath(job, name
          + codec.getDefaultExtension());
      FileSystem fs = file.getFileSystem(job.getConf());
      FSDataOutputStream fileOut = fs.create(file);
      return new LineRecordWriter<K, V>(new DataOutputStream(codec
          .createOutputStream(fileOut)), keyValueSeparator);
    }
  }
}
View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()

      String fileName = indir + "/" + "part-r-" + padZeros(fi, 5) + extension;
      OutputStream filestream = FileUtils.create(fileName);
      BufferedWriter filewriter;
      if (codec != null) {
        filewriter = new BufferedWriter(new OutputStreamWriter(
            codec.createOutputStream(filestream, codec.createCompressor()),
            Data.CHARSET_NAME));
      } else {
        filewriter = new BufferedWriter(new OutputStreamWriter(
            filestream, Data.CHARSET_NAME));
      }
View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec.createOutputStream()

      boolean isCompressed) throws IOException {
    if (isCompressed) {
      Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jc,
          DefaultCodec.class);
      CompressionCodec codec = ReflectionUtils.newInstance(codecClass, jc);
      return codec.createOutputStream(out);
    } else {
      return (out);
    }
  }
View Full Code Here

Examples of org.apache.hadoop.io.compress.DefaultCodec.createOutputStream()

  @Test
  public void testDecompressedStream() throws Exception {
    DefaultCodec codec = new DefaultCodec();
    codec.setConf(fs.getConf());
    String name = "local/" + UUID.randomUUID() + codec.getDefaultExtension();
    OutputStream outputStream = codec.createOutputStream(fs.create(new Path(name)));
    byte[] content = name.getBytes();
    outputStream.write(content);
    outputStream.close();

    Resource resource = loader.getResource(name);
View Full Code Here

Examples of org.apache.hadoop.io.compress.DefaultCodec.createOutputStream()

  public void testCompressedStream() throws Exception {

    DefaultCodec codec = new DefaultCodec();
    codec.setConf(fs.getConf());
    String name = "local/" + UUID.randomUUID() + codec.getDefaultExtension();
    OutputStream outputStream = codec.createOutputStream(fs.create(new Path(name)));
    byte[] content = name.getBytes();
    outputStream.write(content);
    outputStream.close();

    loader.setUseCodecs(false);
View Full Code Here

Examples of org.apache.hadoop.io.compress.GzipCodec.createOutputStream()

        CompressionOutputStream outputStream = Mockito
                .mock(CompressionOutputStream.class);
        Mockito.when(codec.createInputStream(Mockito.any(InputStream.class)))
                .thenReturn(inputStream);

        Mockito.when(codec.createOutputStream(Mockito.any(OutputStream.class)))
                .thenReturn(outputStream);
    }

    private void mockSequenceFileReaderWriter(boolean isCompressed)
            throws Exception {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.