Examples of org.apache.hadoop.mapred.OutputCollector

org.apache.hadoop.mapred.OutputCollector
Collects the <key, value> pairs output by {@link Mapper}s and {@link Reducer}s.
OutputCollector is the generalization of the facility provided by the Map-Reduce framework to collect data output by either the Mapper or the Reducer i.e. intermediate outputs or the output of the job.


    @Override
    public void sink( FlowProcess<JobConf> flowProcess, SinkCall<Object[], OutputCollector> sinkCall ) throws IOException {
        // it's ok to use NULL here so the collector does not write anything
        TupleEntry tupleEntry = sinkCall.getOutgoingEntry();
        OutputCollector outputCollector = sinkCall.getOutput();
        if( updateBy != null )
        {
            Tuple allValues = tupleEntry.selectTuple( updateValueFields );
            Tuple updateValues = tupleEntry.selectTuple( updateByFields );


            allValues = cleanTuple( allValues );


            TupleRecord key = new TupleRecord( allValues );


            if( updateValues.equals( updateIfTuple ) )
                outputCollector.collect( key, null );
            else
                outputCollector.collect( key, key );


            return;
        }


        Tuple result = tupleEntry.selectTuple( getSinkFields() );


        result = cleanTuple( result );


        outputCollector.collect( new TupleRecord( result ), null );
    }

View Full Code Here


  @Override
  public void sink(FlowProcess<JobConf> flowProcess, SinkCall<Object[], OutputCollector> sinkCall)
      throws IOException {
    TupleEntry tupleEntry = sinkCall.getOutgoingEntry();
    OutputCollector outputCollector = sinkCall.getOutput();
    Tuple key = tupleEntry.selectTuple(keyField);
    ImmutableBytesWritable keyBytes = (ImmutableBytesWritable) key.getObject(0);
    Put put = new Put(keyBytes.get());


    for (int i = 0; i < valueFields.length; i++) {
      Fields fieldSelector = valueFields[i];
      TupleEntry values = tupleEntry.selectEntry(fieldSelector);


      for (int j = 0; j < values.getFields().size(); j++) {
        Fields fields = values.getFields();
        Tuple tuple = values.getTuple();


        ImmutableBytesWritable valueBytes = (ImmutableBytesWritable) tuple.getObject(j);
        put.add(Bytes.toBytes(familyNames[i]), Bytes.toBytes((String) fields.get(j)), valueBytes.get());
      }
    }


    outputCollector.collect(null, put);
  }

View Full Code Here

        // called at all in this task). If reducer still generates output,
        // which is very uncommon and we may not have to support this case.
        // So we don't write this output to HDFS, but we consume/collect
        // this output just to avoid reducer hanging forever.


        OutputCollector collector = new OutputCollector() {
          public void collect(Object key, Object value)
            throws IOException {
            //just consume it, no need to write the record anywhere
          }
        };

View Full Code Here

    final OutputCollector output, Reporter r)
    throws IOException
  {
    final String collection = Nutchwax.getCollectionFromWaxKey(key);
    
    final OutputCollector oo = new OutputCollector()
    {
      public void collect(WritableComparable k, Writable v)
        throws IOException
      {
        output.collect(Nutchwax.generateWaxKey(k, collection), v);

View Full Code Here


  public void reduce(final WritableComparable key, Iterator values,
    final OutputCollector output, Reporter reporter)
    throws IOException
  {
    OutputCollector oc = new OutputCollector()
    {
      public void collect(WritableComparable k, Writable v)
        throws IOException
      {
        // Substitute original key in place of passed k.  The original

View Full Code Here

    final OutputCollector output, Reporter r)
    throws IOException
  {
    final String collection = Nutchwax.getCollectionFromWaxKey(key);
    
    final OutputCollector oo = new OutputCollector()
    {
      public void collect(WritableComparable k, Writable v)
        throws IOException
      {
        output.collect(Nutchwax.generateWaxKey(k, collection), v);

View Full Code Here

        // called at all in this task). If reducer still generates output,
        // which is very uncommon and we may not have to support this case.
        // So we don't write this output to HDFS, but we consume/collect
        // this output just to avoid reducer hanging forever.


        OutputCollector collector = new OutputCollector() {
          public void collect(Object key, Object value)
            throws IOException {
            //just consume it, no need to write the record anywhere
          }
        };

View Full Code Here

        // create a new record writer
        writer = outputFormat.getRecordWriter(FileSystem.get(outputConf), 
                                              outputConf, fileName, reporter);


        // return an output collector using the writer we just created.
        return new StoreFuncAdaptor(new OutputCollector() 
            {
                public void collect(Object key, Object value) throws IOException {
                    writer.write(key,value);
                }
            });

View Full Code Here

   * Handle the end of the input by closing down the application.
   */
  public void close() throws IOException {
    // if we haven't started the application, we have nothing to do
    if (isOk) {
      OutputCollector nullCollector = new OutputCollector() {
        public void collect(WritableComparable key, 
                            Writable value) throws IOException {
          // NULL
        }
      };

View Full Code Here

  private void runOldCombiner(final TezRawKeyValueIterator rawIter, final Writer writer) throws IOException {
    Class<? extends Reducer> reducerClazz = (Class<? extends Reducer>) conf.getClass("mapred.combiner.class", null, Reducer.class);
    
    Reducer combiner = ReflectionUtils.newInstance(reducerClazz, conf);
    
    OutputCollector collector = new OutputCollector() {
      @Override
      public void collect(Object key, Object value) throws IOException {
        writer.append(key, value);
      }
    };

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.mapred.OutputCollector

cascading.flow.tez.stream.element.TezBoundaryStage

cascading.flow.tez.stream.element.TezMergeGate

com.twitter.elephantbird.cascading2.scheme.LzoBinaryScheme

com.twitter.maple.hbase.HBaseScheme

com.twitter.maple.jdbc.JDBCScheme

org.apache.hadoop.hbase.mapred.TestTableSnapshotInputFormat

org.apache.hadoop.mapred.pipes.PipesReducer

org.apache.hadoop.streaming.PipeMapRed

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReducePOStoreImpl

org.apache.tez.mapreduce.combine.MRCombiner

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.