Examples of FileOutputCommitter

org.apache.hadoop.mapred.FileOutputCommitter
An {@link OutputCommitter} that commits files specified in job output directory i.e. ${mapred.output.dir}.
org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
An {@link OutputCommitter} that commits files specified in job output directory i.e. ${mapreduce.output.fileoutputformat.outputdir}.

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

  public void checkOutputSpecs(JobContext context) 
      throws IOException, InterruptedException {}


  public OutputCommitter getOutputCommitter(TaskAttemptContext context) 
      throws IOException, InterruptedException {
    return new FileOutputCommitter(FileOutputFormat.getOutputPath(context),
                                   context);
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

            if (components.contains(committer) == false) {
                components.add(committer);
            }
        }
        if (isTemporaryOutputEnabled(context)) {
            FileOutputCommitter committer = temporaryOutputFormat.getOutputCommitter(context);
            if (components.contains(committer) == false) {
                components.add(committer);
            }
        }
        if (LOG.isDebugEnabled()) {

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

        Configuration conf = context.getConfiguration();
        if (FileOutputFormat.getCompressOutput(context)) {
            Class<?> codecClass = FileOutputFormat.getOutputCompressorClass(context, DefaultCodec.class);
            codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
        }
        FileOutputCommitter committer = getOutputCommitter(context);
        Path file = new Path(
                committer.getWorkPath(),
                FileOutputFormat.getUniqueFile(context, name, ""));
        final ModelOutput<V> out = TemporaryStorage.openOutput(conf, dataType, file, codec);
        return new RecordWriter<NullWritable, V>() {


            @Override

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

    private FileOutputCommitter createOutputCommitter(TaskAttemptContext context) throws IOException {
        assert context != null;
        if (getOutputPath(context).equals(FileOutputFormat.getOutputPath(context))) {
            return (FileOutputCommitter) new EmptyFileOutputFormat().getOutputCommitter(context);
        } else {
            return new FileOutputCommitter(getOutputPath(context), context);
        }
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter


  public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context)
  throws IOException, InterruptedException {
    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    final Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
        HConstants.DEFAULT_MAX_FILE_SIZE);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter


    /** Default implementation assumes FileOutputFormat. Storage drivers wrapping
     * other OutputFormats should override this method.
     */
    public Path getWorkFilePath(TaskAttemptContext context, String outputLoc) throws IOException{
      return new Path(new FileOutputCommitter(new Path(outputLoc), context).getWorkPath(), FileOutputFormat.getUniqueFile(context, "part",""));
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

  public org.apache.hadoop.mapreduce.RecordWriter<WritableComparable<?>, BytesRefArrayWritable> getRecordWriter(
      TaskAttemptContext task) throws IOException, InterruptedException {


    //FileOutputFormat.getWorkOutputPath takes TaskInputOutputContext instead of
    //TaskAttemptContext, so can't use that here
    FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter(task);
    Path outputPath = committer.getWorkPath();


    FileSystem fs = outputPath.getFileSystem(task.getConfiguration());


    if (!fs.exists(outputPath)) {
      fs.mkdirs(outputPath);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

    String outputPath = context.getConfiguration().get("mapred.output.dir");
    //we need to do this to get the task path and set it for mapred implementation
    //since it can't be done automatically because of mapreduce->mapred abstraction
    if (outputPath != null)
      context.getConfiguration().set("mapred.work.output.dir",
        new FileOutputCommitter(new Path(outputPath), context).getWorkPath().toString());
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter


    /** Default implementation assumes FileOutputFormat. Storage drivers wrapping
     * other OutputFormats should override this method.
     */
    public Path getWorkFilePath(TaskAttemptContext context, String outputLoc) throws IOException{
      return new Path(new FileOutputCommitter(new Path(outputLoc), context).getWorkPath(), FileOutputFormat.getUniqueFile(context, "part",""));
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

  public org.apache.hadoop.mapreduce.RecordWriter<WritableComparable<?>, BytesRefArrayWritable> getRecordWriter(
      TaskAttemptContext task) throws IOException, InterruptedException {


    //FileOutputFormat.getWorkOutputPath takes TaskInputOutputContext instead of
    //TaskAttemptContext, so can't use that here
    FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter(task);
    Path outputPath = committer.getWorkPath();


    FileSystem fs = outputPath.getFileSystem(task.getConfiguration());


    if (!fs.exists(outputPath)) {
      fs.mkdirs(outputPath);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.