Examples of FileOutputCommitter

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

50
51
52
53
54
55
56
57
58
59
60
public class HFileOutputFormat extends FileOutputFormat<ImmutableBytesWritable, KeyValue> {
  public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(TaskAttemptContext context)
  throws IOException, InterruptedException {
    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong("hbase.hregion.max.filesize", 268435456);
    final int blocksize = conf.getInt("hfile.min.blocksize.size", 65536);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

237
238
239
240
241
242
243
244
  }

  /** (@inheritDoc) */
  public OutputCommitter getOutputCommitter(TaskAttemptContext context)
      throws IOException, InterruptedException {
    return new FileOutputCommitter(FileOutputFormat.getOutputPath(context),
        context);
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

237
238
239
240
241
242
243
        String outputPath = context.getConfiguration().get("mapred.output.dir");
        //we need to do this to get the task path and set it for mapred implementation
        //since it can't be done automatically because of mapreduce->mapred abstraction
        if(outputPath != null)
            context.getConfiguration().set("mapred.work.output.dir",
                    new FileOutputCommitter(new Path(outputPath), context).getWorkPath().toString());
    }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

70
71
72
73
74
75
76
77
78
79
80
  public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(
      final TaskAttemptContext context) throws IOException, InterruptedException {

    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();

    final Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);

    // These configs. are from hbase-*.xml
 
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

66
67
68
69
70
71
72
  @Override
  public OutputCommitter getOutputCommitter(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException
  {
    // Is there a programmatic way to get the temp dir? I see it hardcoded everywhere in Hadoop, Hive, and Pig.
    return new FileOutputCommitter(new Path("/tmp/" + taskAttemptContext.getTaskAttemptID().getJobID().toString()), taskAttemptContext);
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

246
247
248
249
250
251
252
    String outputPath = context.getConfiguration().get("mapred.output.dir");
    //we need to do this to get the task path and set it for mapred implementation
    //since it can't be done automatically because of mapreduce->mapred abstraction
    if (outputPath != null)
      context.getConfiguration().set("mapred.work.output.dir",
        new FileOutputCommitter(new Path(outputPath), context).getWorkPath().toString());
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

60
61
62
63
64
65
66
67
68
69
70
71
  public org.apache.hadoop.mapreduce.RecordWriter<WritableComparable<?>, BytesRefArrayWritable> getRecordWriter(
    TaskAttemptContext task) throws IOException, InterruptedException {

    //FileOutputFormat.getWorkOutputPath takes TaskInputOutputContext instead of
    //TaskAttemptContext, so can't use that here
    FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter(task);
    Path outputPath = committer.getWorkPath();

    FileSystem fs = outputPath.getFileSystem(task.getConfiguration());

    if (!fs.exists(outputPath)) {
      fs.mkdirs(outputPath);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

246
247
248
249
250
251
252
    String outputPath = context.getConfiguration().get("mapred.output.dir");
    //we need to do this to get the task path and set it for mapred implementation
    //since it can't be done automatically because of mapreduce->mapred abstraction
    if (outputPath != null)
      context.getConfiguration().set("mapred.work.output.dir",
        new FileOutputCommitter(new Path(outputPath), context).getWorkPath().toString());
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

114
115
116
117
118
119
120
121
122
123
124
      createRecordWriter(final TaskAttemptContext context)
          throws IOException, InterruptedException {

    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    final Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
        HConstants.DEFAULT_MAX_FILE_SIZE);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

55
56
57
58
59
60
61
62
  public void checkOutputSpecs(JobContext context)
      throws IOException, InterruptedException {}

  public OutputCommitter getOutputCommitter(TaskAttemptContext context)
      throws IOException, InterruptedException {
    return new FileOutputCommitter(FileOutputFormat.getOutputPath(context),
                                   context);
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.