Examples of MultipleOutputs

org.apache.hadoop.mapred.lib.MultipleOutputs

nes additional single text based output 'text' for the job MultipleOutputs.addNamedOutput(conf, "text", TextOutputFormat.class, LongWritable.class, Text.class); // Defines additional multi sequencefile based output 'sequence' for the // job MultipleOutputs.addMultiNamedOutput(conf, "seq", SequenceFileOutputFormat.class, LongWritable.class, Text.class); ... JobClient jc = new JobClient(); RunningJob job = jc.submitJob(conf); ...

Job configuration usage pattern is:

 public class MOReduce implements Reducer<WritableComparable, Writable> { private MultipleOutputs mos; public void configure(JobConf conf) { ... mos = new MultipleOutputs(conf); } public void reduce(WritableComparable key, Iterator<Writable> values, OutputCollector output, Reporter reporter) throws IOException { ... mos.getCollector("text", reporter).collect(key, new Text("Hello")); mos.getCollector("seq", "A", reporter).collect(key, new Text("Bye")); mos.getCollector("seq", "B", reporter).collect(key, new Text("Chau")); ... } public void close() throws IOException { mos.close(); ... } }

@deprecated Use {@link org.apache.hadoop.mapreduce.lib.output.MultipleOutputs} instead

org.apache.hadoop.mapreduce.lib.output.MultipleOutputs

nes additional single text based output 'text' for the job MultipleOutputs.addNamedOutput(job, "text", TextOutputFormat.class, LongWritable.class, Text.class); // Defines additional sequence-file based output 'sequence' for the job MultipleOutputs.addNamedOutput(job, "seq", SequenceFileOutputFormat.class, LongWritable.class, Text.class); ... job.waitForCompletion(true); ...

Usage in Reducer:

  String generateFileName(K k, V v) { return k.toString() + "_" + v.toString(); } public class MOReduce extends Reducer<WritableComparable, Writable,WritableComparable, Writable> { private MultipleOutputs mos; public void setup(Context context) { ... mos = new MultipleOutputs(context); } public void reduce(WritableComparable key, Iterator<Writable> values, Context context) throws IOException { ... mos.write("text", , key, new Text("Hello")); mos.write("seq", LongWritable(1), new Text("Bye"), "seq_a"); mos.write("seq", LongWritable(2), key, new Text("Chau"), "seq_b"); mos.write(key, new Text("value"), generateFileName(key, new Text("value"))); ... } public void cleanup(Context) throws IOException { mos.close(); ... } }

Examples of org.apache.hadoop.mapred.lib.MultipleOutputs

       _props = KafkaETLUtils.getPropsFromJob(job);
       _contextList = new ArrayList<KafkaETLContext>();
       _job = job;
       _reporter = reporter;
       _contextIndex = -1;
       _mos = new MultipleOutputs(job);
       try {
           _limit = _props.getInt("kafka.request.limit", -1);
           
           /*get attemp id*/
           String taskId = _job.get("mapred.task.id");

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.MultipleOutputs


    @SuppressWarnings({ "unchecked", "rawtypes" })
    @Override
    protected void setup(Context context) {
      // Create a new MultipleOutputs using the context object
      mos = new MultipleOutputs(context);
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.MultipleOutputs

    private final Reducer.Context context;
    private final boolean testing;


    public SafeReducerOutputs(final Reducer.Context context) {
        this.context = context;
        this.outputs = new MultipleOutputs(this.context);
        this.testing = this.context.getConfiguration().getBoolean(FaunusCompiler.TESTING, false);
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.MultipleOutputs

    private final Mapper.Context context;
    private final boolean testing;


    public SafeMapperOutputs(final Mapper.Context context) {
        this.context = context;
        this.outputs = new MultipleOutputs(this.context);
        this.testing = this.context.getConfiguration().getBoolean(FaunusCompiler.TESTING, false);
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.MultipleOutputs

  MSCR mscr; // Current MSCR being executed
  
  protected void setup(Reducer<PlumeObject, PlumeObject, NullWritable, NullWritable>.Context context)
    throws IOException, InterruptedException {


    this.mos  = new MultipleOutputs(context);
    this.mscr = MapRedExecutor.readMSCR(context.getConfiguration());
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.MultipleOutputs


    private MultipleOutputs mos;


    @Override
    protected void setup(Context context) {
      mos = new MultipleOutputs(context);
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.MultipleOutputs


    private MultipleOutputs mos;


    @Override
    protected void setup(Context context) {
      mos = new MultipleOutputs(context);
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.MultipleOutputs

    private float randomSelectionPercent;


    @Override
    protected void setup(Context ctx) throws IOException {
      randomSelectionPercent = ctx.getConfiguration().getFloat(RANDOM_SELECTION_PCT, 0);
      multipleOutputs = new MultipleOutputs(ctx);
    }

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.