Examples of FileOutputCommitter

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

55
56
57
58
59
60
61
62
  public void checkOutputSpecs(JobContext context)
      throws IOException, InterruptedException {}

  public OutputCommitter getOutputCommitter(TaskAttemptContext context)
      throws IOException, InterruptedException {
    return new FileOutputCommitter(FileOutputFormat.getOutputPath(context),
                                   context);
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

77
78
79
80
81
82
83
84
85
86
87
  public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context)
  throws IOException, InterruptedException {
    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    final Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong("hbase.hregion.max.filesize",
        HConstants.DEFAULT_MAX_FILE_SIZE);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

68
69
70
71
72
73
74
75
76
77
78
 
  public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context)
  throws IOException, InterruptedException {
    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong("hbase.hregion.max.filesize", 268435456);
    final int blocksize = conf.getInt("hfile.min.blocksize.size", 65536);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

85
86
87
88
89
90
91
92
93
94
95
  public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context)
  throws IOException, InterruptedException {
    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    final Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
        HConstants.DEFAULT_MAX_FILE_SIZE);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

194
195
196
197
198
199
200
201
202
203
204
205
    String outputPath = getOutputDir(conf);
    // we need to do this to get the task path and set it for mapred
    // implementation since it can't be done automatically because of
    // mapreduce->mapred abstraction
    if (outputPath != null) {
      FileOutputCommitter foc = new FileOutputCommitter(getOutputPath(conf), context);
      Path path = foc.getWorkPath();
      FileSystem fs = path.getFileSystem(conf);
      fs.mkdirs(path);
      conf.set("mapred.work.output.dir", path.toString());
      LOG.info("Setting mapred.work.output.dir to {}", path.toString());
    }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

102
103
104
105
106
107
108
109
110
 
  public OutputCommitter getOutputCommitter(TaskAttemptContext context)
      throws IOException {
    if (committer == null) {
      Path output = getOutputPath(context);
      committer = new FileOutputCommitter(output, context);
    }
    return committer;
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

916
917
918
919
920
921
922
923
924
925
        }

        @Override
        public Path getDefaultWorkFile(TaskAttemptContext context,
                String extension) throws IOException {
            FileOutputCommitter committer =
                    (FileOutputCommitter) super.getOutputCommitter(context);
            return new Path(committer.getWorkPath(), getUniqueFile(context,
                    "part", extension));
        }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

116
117
118
119
120
121
122
123
124
125
126
      createRecordWriter(final TaskAttemptContext context)
          throws IOException, InterruptedException {

    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    final Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
        HConstants.DEFAULT_MAX_FILE_SIZE);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

175
176
177
178
179
180
181
182
183
184
    String outputPath = getOutputDir(conf);
    // we need to do this to get the task path and set it for mapred
    // implementation since it can't be done automatically because of
    // mapreduce->mapred abstraction
    if (outputPath != null) {
      FileOutputCommitter foc =
          new FileOutputCommitter(getOutputPath(conf), context);
      conf.set("mapred.work.output.dir", foc.getWorkPath().toString());
    }
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

208
209
210
211
212
213
214
215
216
217
218
                        HCatMapRedUtil.createTaskAttemptContext(currJobContext.getJobConf(),
                                                                                        currTaskContext.getTaskAttemptID(),
                                                                                        currTaskContext.getProgressible());
                //set temp location
                currTaskContext.getConfiguration().set("mapred.work.output.dir",
                                new FileOutputCommitter(new Path(localJobInfo.getLocation()),currTaskContext).getWorkPath().toString());
                //setupTask()
                baseOutputCommitter.setupTask(currTaskContext);

                org.apache.hadoop.mapred.RecordWriter baseRecordWriter =
                        baseOF.getRecordWriter(null,
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.