Examples of cascading.tuple.TupleEntryCollector

cascading.tuple.TupleEntryCollector
Interface TupleEntryCollector is used to allow {@link cascading.operation.BaseOperation} instances to emitone or more result {@link Tuple} values.
The general rule in Cascading is if you are handed a Tuple, you cannot change or cache it. Attempts at modifying such a Tuple will result in an Exception. Preventing caching is harder, see below.
If you create the Tuple, you can re-use or modify it.
When calling {@link #add(Tuple)} or {@link #add(TupleEntry)}, you are passing a Tuple to the down stream pipes and operations. Since no downstream operation may modify or cache the Tuple instance, it is safe to re-use the Tuple instance when {@code add()} returns.
That said, Tuple copies do get cached in order to perform specific operations in the underlying platforms. Currently only a shallow copy is made (via the {@link Tuple} copy constructor). Thus, any mutable type or collectionplaced inside a Tuple will not be copied, but will likely be cached if a copy of the Tuple passed downstream is copied.
So any subsequent changes to that nested type or collection will be reflected in the cached copy, a likely source of hard to find errors.
There is currently no way to specify that a deep copy must be performed when making a Tuple copy.

  }


  public void operate(FlowProcess flow_process, BufferCall call) {
    ISeq resultSeq =
        RT.seq(invokeFunction(new TupleSeqConverter(call.getArgumentsIterator())));
    TupleEntryCollector collector = call.getOutputCollector();
    while (resultSeq != null) {
      Object obj = resultSeq.first();
      collector.add(Util.coerceToTuple(obj));
      resultSeq = resultSeq.next();
    }
  }

View Full Code Here

    super(groupFields, false, null, argFields, outFields, agg_specs, "cascalog.combiner.aggregator.size", 10000);
  }


  @Override
  protected void write(Tuple group, List<Object> val, OperationCall opCall) {
    TupleEntryCollector output = ((FunctionCall) opCall).getOutputCollector();
    Tuple t = new Tuple(group);
    for (Object o : val) {
      t.add(o);
    }
    output.add(t);
  }

View Full Code Here

    extract_fn = Util.deserializeFn(spec.presentFn);
  }


  @Override
  protected void write(Tuple group, List<Object> vals, OperationCall opCall) {
    TupleEntryCollector output = ((FunctionCall) opCall).getOutputCollector();


    if (vals.size() != 1) {
      throw new RuntimeException(
          "Should only have one object in buffer combiner before extraction " + vals.size() + ":"
          + vals.toString());
    }
    Object val = vals.get(0);
    try {
      ISeq result_seq = RT.seq(extract_fn.invoke(val));
      while (result_seq != null) {
        Tuple t = Util.coerceToTuple(result_seq.first());
        Tuple emit = new Tuple(group);
        emit.addAll(t);
        output.add(emit);
        result_seq = result_seq.next();
      }
    } catch (Exception e) {
      throw new RuntimeException(e);
    }

View Full Code Here

  }


  @Override
  public void operate(FlowProcess flowProcess, FunctionCall functionCall) {
    Object inputTuple = convertInput(functionCall.getArguments());
    TupleEntryCollector outputCollector = functionCall.getOutputCollector();


    callArgs[0] = Py.java2py(inputTuple);
    if (outputMethod == OutputMethod.COLLECTS) {
      // The Python function collects the output tuples itself into the output
      // collector

View Full Code Here

    // This gets called even when there are no tuples in the group after
    // a GroupBy (see the Buffer javadoc). So we need to check if there are any
    // valid tuples returned in the group.
    if (arguments.hasNext()) {
      TupleEntry group = bufferCall.getGroup();
      TupleEntryCollector outputCollector = bufferCall.getOutputCollector();


      callArgs[0] = Py.java2py(group);
      callArgs[1] = Py.java2py(arguments);
      if (outputMethod == OutputMethod.COLLECTS) {
        callArgs[2] = Py.java2py(outputCollector);

View Full Code Here

  }


  @Override
  public void operate(FlowProcess flowProcess, FunctionCall functionCall) {
    TupleEntry inputTuple = functionCall.getArguments();
    TupleEntryCollector outputCollector = functionCall.getOutputCollector();
    Tuple outputTuple = new Tuple();


    for (Comparable field : filteredFields) {
      // We cannot use inputTuple.get(...) here, as that tries to convert
      // the field value to a Comparable. In case we have a complex Python
      // type as a field, that won't work.
      outputTuple.add(inputTuple.getObject(field));
    }
    outputCollector.add(outputTuple);
  }

View Full Code Here

  }


  @Override
  public void aggregate(FlowProcess flowProcess, AggregatorCall aggregatorCall) {
    TupleEntry group = aggregatorCall.getGroup();
    TupleEntryCollector outputCollector = aggregatorCall.getOutputCollector();


    System.out.println("Aggregator called with group: " + group);
  }

View Full Code Here

    runTest( tap );
    }


  private void runTest( Tap tap ) throws IOException
    {
    TupleEntryCollector collector = tap.openForWrite( getPlatform().getFlowProcess() ); // casting for test


    for( int i = 0; i < 100; i++ )
      collector.add( new Tuple( "string", "" + i, i ) );


    collector.close();


    TupleEntryIterator iterator = tap.openForRead( getPlatform().getFlowProcess() );


    int count = 0;
    while( iterator.hasNext() )

View Full Code Here


      @Override
      public void operate( FunctionCall<Functor> functionCall )
        {
        TupleEntry input = functionCall.getArguments();
        TupleEntryCollector outputCollector = functionCall.getOutputCollector();


        outputCollector.add( input.getCoercedTuple( types, result ) );
        }
      };
      }
    else
      {
      functor = new Functor()
      {
      @Override
      public void operate( FunctionCall<Functor> functionCall )
        {
        TupleEntryCollector outputCollector = functionCall.getOutputCollector();


        outputCollector.add( functionCall.getArguments().getTuple() );
        }
      };
      operationCall.setContext( functor );
      }

View Full Code Here

      this.pathFields = ( (TemplateScheme) getScheme() ).pathFields;
      }


    private TupleEntryCollector getCollector( String path )
      {
      TupleEntryCollector collector = collectors.get( path );


      if( collector != null )
        return collector;


      try

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of cascading.tuple.TupleEntryCollector

bixo.datum.ScoredUrlDatumTest

bixo.examples.crawl.CreateUrlDatumFromOutlinksFunction

bixo.examples.crawl.DemoCrawlTool

bixo.examples.crawl.LatestUrlDatumBufferTest

bixo.examples.webmining.CreateLinkDatumFromOutlinksFunction

bixo.examples.webmining.CreateResultsFunction

bixo.examples.webmining.DemoWebMiningWorkflow

bixo.operations.FilterAndScoreByUrlAndRobotsTest

bixo.operations.MakeFetchSetsBuffer

bixo.pipes.AbstractFetchPipeTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.