Examples of cascading.pipe.Every

cascading.pipe.Every
The Every operator applies an {@link Aggregator} or {@link Buffer} to every grouping.
Any number of Every instances may follow other Every, {@link GroupBy}, or {@link CoGroup} instances if they apply anAggregator, not a Buffer. If a Buffer, only one Every may follow a GroupBy or CoGroup.
Every operators create aggregate values for every grouping they encounter. This aggregate value is added to the current grouping Tuple.
In the case of a CoGroup, the grouping Tuple will be all the grouping keys from all joined streams, and if an "outer" type join is used, one value on the groupingTuple may be null.
Subsequent Every instances can continue to append values to the grouping Tuple. When an Each follows and Every, the Each applies its operation to the grouping Tuple (thus all child values in the grouping are discarded and only aggregate values are propagated).


    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );


    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Every( pipe, new Fields( "ip" ), new Count(), new Fields( "ip", "count" ) );


    pipe = new Each( pipe, new Fields( "ip" ), new RegexFilter( "^68.*" ) );


    Pipe left = new Each( pipe, new Fields( "ip" ), new RegexFilter( ".*46.*" ) );

View Full Code Here

    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );
    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter );


    Pipe splice = new CoGroup( pipeLower, new Fields( "num" ), pipeUpper, new Fields( "num" ), new Fields( "num1", "lhs", "num2", "rhs" ) );


    splice = new Every( splice, new First( new Fields( "value" ) ), new Fields( "num1", "value" ) );


    Flow countFlow = getPlatform().getFlowConnector().connect( sources, sink, splice );
    }

View Full Code Here

    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );
    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter );


    Pipe splice = new CoGroup( pipeLower, new Fields( "num" ), pipeUpper, new Fields( "num" ), new Fields( "num1", "lhs", "num2", "rhs" ), new Fields( "somenum", "somenum2" ) );


    splice = new Every( splice, new First( new Fields( "value" ) ), new Fields( "somenum", "value" ) );


    Flow countFlow = getPlatform().getFlowConnector().connect( sources, sink, splice );
    }

View Full Code Here

    Pipe pipeLower1 = new Pipe( "lower1" );
    Pipe pipeUpper1 = new Pipe( "upper1" );


    Pipe pipeLower2 = new Each( pipeLower1, new Identity() );
    pipeLower2 = new GroupBy( pipeLower2, new Fields( "num", "char" ) );
    pipeLower2 = new Every( pipeLower2, new Fields( "num", "char" ), new Count(), new Fields( "num", "char" ) );


    pipeLower1 = new Each( pipeLower1, new Identity() );
    pipeLower1 = new Each( pipeLower1, new Identity() );
    pipeLower1 = new Each( pipeLower1, new Identity() );
    pipeLower1 = new Pipe( "lower2", pipeLower1 );

View Full Code Here

    sinks.put( "output2", sink2 );


    Pipe pipeLower1 = new Pipe( "lower1" );


    Pipe pipe = new GroupBy( pipeLower1, new Fields( 0 ) );
    Pipe left = new Every( new Pipe( "output1", pipe ), new TestBuffer( new Fields( "left" ), true ) );
    Pipe right = new Every( new Pipe( "output2", pipe ), new TestBuffer( new Fields( "right" ), true ) );


    Flow flow = null;
    try
      {
      flow = getPlatform().getFlowConnector().connect( sources, sinks, Pipe.pipes( left, right ) );

View Full Code Here

    sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) );
    sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    Pipe pipe = new Pipe( "count" );
    pipe = new GroupBy( pipe, new Fields( 1 ) );
    pipe = new Every( pipe, new Fields( 1 ), new TestBuffer( new Fields( "fourth" ), "value" ), new Fields( 0, 1 ) );


    List steps = getPlatform().getFlowConnector().connect( sources, sinks, pipe ).getFlowSteps();


    assertEquals( "wrong size", 1, steps.size() );

View Full Code Here

    sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) );
    sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    Pipe pipe = new Pipe( "count" );
    pipe = new GroupBy( pipe, new Fields( 1 ) );
    pipe = new Every( pipe, new Fields( 1 ), new TestBuffer( new Fields( "fourth" ), "value" ), new Fields( 0, 1 ) );
    pipe = new Every( pipe, new Fields( 1 ), new Count(), new Fields( 0, 1 ) );


    try
      {
      getPlatform().getFlowConnector().connect( sources, sinks, pipe );
      fail( "did not throw planner exception" );

View Full Code Here

    sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) );
    sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    Pipe pipe = new Pipe( "count" );
    pipe = new GroupBy( pipe, new Fields( 1 ) );
    pipe = new Every( pipe, new Fields( 1 ), new Count(), new Fields( 0, 1 ) );
    pipe = new Every( pipe, new Fields( 1 ), new TestBuffer( new Fields( "fourth" ), "value" ), new Fields( 0, 1 ) );


    try
      {
      getPlatform().getFlowConnector().connect( sources, sinks, pipe );
      fail( "did not throw planner exception" );

View Full Code Here

    Pipe right = new Each( new Pipe( "right" ), new Fields( "line" ), new RegexFilter( ".*192.*" ) );


    Fields[] fields = Fields.fields( new Fields( "offset" ), new Fields( "offset" ) );
    Pipe merge = new HashJoin( "join", Pipe.pipes( left, right ), fields, Fields.size( 4 ), new InnerJoin() );


    merge = new Every( merge, new MaxValue() );


    Map sources = new HashMap();
    sources.put( "left", source1 );
    sources.put( "right", source2 );

View Full Code Here

      count += 2; // 2 taps


      if( i > 0 )
        {
        pipe = new CoGroup( pipes[ i - 1 ], new Fields( "key" + ( i - 1 ) ), pipe, new Fields( "key" + i ) );
        pipe = new Every( pipe, new Fields( "key" + ( i - 1 ) ), new Sum() );
        count += 2; // 2 pipes
        }


      pipes[ i ] = new Pipe( nameOut, pipe );
      count += 1; // 1 pipe

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of cascading.pipe.Every

bixo.examples.crawl.DemoCrawlWorkflow

bixo.examples.crawl.LatestUrlDatumBufferTest

bixo.examples.webmining.DemoWebMiningWorkflow

bixo.pipes.FetchPipe

cascading.assembly.CrossTab

cascading.BasicPipesPlatformTest

cascading.BasicTrapPlatformTest

cascading.BufferPipesPlatformTest

cascading.CoGroupFieldedPipesPlatformTest

cascading.detail.EveryAssemblyFactory

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.