Examples of FileOutputCommitter

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

85
86
87
88
89
90
91
92
93
94
95
  public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context)
  throws IOException, InterruptedException {
    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    final Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
        HConstants.DEFAULT_MAX_FILE_SIZE);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

87
88
89
90
91
92
93
94
95
96
97
  public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context)
  throws IOException, InterruptedException {
    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    final Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
        HConstants.DEFAULT_MAX_FILE_SIZE);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

68
69
70
71
72
73
74
75
76
77
78
  public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context)
  throws IOException, InterruptedException {
    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong("hbase.hregion.max.filesize", 268435456);
    final int blocksize =
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

138
139
140
141
142
143
144
145
146
147
148
   
    // Setup the Mapper
    WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(tmpFile, 0, fs.pathToFile(tmpFile).length(), null),0);
    AggregatingRecordReader rr = new AggregatingRecordReader();
    Path ocPath = new Path(tmpFile, "oc");
    OutputCommitter oc = new FileOutputCommitter(ocPath, context);
    fs.deleteOnExit(ocPath);
    StandaloneStatusReporter sr = new StandaloneStatusReporter();
    rr.initialize(split, context);
    MockAccumuloRecordWriter rw = new MockAccumuloRecordWriter();
    WikipediaMapper mapper = new WikipediaMapper();
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

131
132
133
134
135
136
137
138
139
140
141
   
    // Setup the Mapper
    InputSplit split = new FileSplit(tmpFile, 0, fs.pathToFile(tmpFile).length(), null);
    AggregatingRecordReader rr = new AggregatingRecordReader();
    Path ocPath = new Path(tmpFile, "oc");
    OutputCommitter oc = new FileOutputCommitter(ocPath, context);
    fs.deleteOnExit(ocPath);
    StandaloneStatusReporter sr = new StandaloneStatusReporter();
    rr.initialize(split, context);
    MockAccumuloRecordWriter rw = new MockAccumuloRecordWriter();
    WikipediaMapper mapper = new WikipediaMapper();
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

96
97
98
99
100
101
102
103
104
105
106
  static <V extends Cell> RecordWriter<ImmutableBytesWritable, V>
      createRecordWriter(final TaskAttemptContext context)
          throws IOException, InterruptedException {
    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    final Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
        HConstants.DEFAULT_MAX_FILE_SIZE);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

240
241
242
243
244
245
246
        String outputPath = context.getConfiguration().get("mapred.output.dir");
        //we need to do this to get the task path and set it for mapred implementation
        //since it can't be done automatically because of mapreduce->mapred abstraction
        if (outputPath != null)
            context.getConfiguration().set("mapred.work.output.dir",
                new FileOutputCommitter(new Path(outputPath), context).getWorkPath().toString());
    }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

212
213
214
215
216
217
218
219
220
221
222
                    HCatMapRedUtil.createTaskAttemptContext(currJobContext.getJobConf(),
                        currTaskContext.getTaskAttemptID(),
                        currTaskContext.getProgressible());
                //set temp location
                currTaskContext.getConfiguration().set("mapred.work.output.dir",
                    new FileOutputCommitter(new Path(localJobInfo.getLocation()), currTaskContext).getWorkPath().toString());
                //setupTask()
                baseOutputCommitter.setupTask(currTaskContext);

                org.apache.hadoop.mapred.RecordWriter baseRecordWriter =
                    baseOF.getRecordWriter(null,
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

60
61
62
63
64
65
66
67
68
69
70
71
    public org.apache.hadoop.mapreduce.RecordWriter<WritableComparable<?>, BytesRefArrayWritable> getRecordWriter(
        TaskAttemptContext task) throws IOException, InterruptedException {

        //FileOutputFormat.getWorkOutputPath takes TaskInputOutputContext instead of
        //TaskAttemptContext, so can't use that here
        FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter(task);
        Path outputPath = committer.getWorkPath();

        FileSystem fs = outputPath.getFileSystem(task.getConfiguration());

        if (!fs.exists(outputPath)) {
            fs.mkdirs(outputPath);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

87
88
89
90
91
92
93
94
95
96
97
  public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context)
  throws IOException, InterruptedException {
    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    final Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
        HConstants.DEFAULT_MAX_FILE_SIZE);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.