Examples of cascading.flow.Flow

cascading.flow.Flow
A Flow is a logical unit of work declared by an assembly of {@link cascading.pipe.Pipe} instances connected to sourceand sink {@link Tap} instances.
A Flow is then executed to push the incoming source data through the assembly into one or more sinks.
A Flow sub-class instance may not be instantiated directly in most cases, see sub-classes of {@link FlowConnector} classfor supported platforms.
Note that {@link cascading.pipe.Pipe} assemblies can be reused in multiple Flow instances. They maintainno state regarding the Flow execution. Subsequently, {@link cascading.pipe.Pipe} assemblies can be givenparameters through its calling Flow so they can be built in a generic fashion.
When a Flow is created, an optimized internal representation is created that is then executed on the underlying execution platform. This is typically done by creating one or more {@link FlowStep} instances.
Flows are submitted in order of dependency when used with a {@link cascading.cascade.Cascade}. If two or more steps do not share the same dependencies and all can be scheduled simultaneously, the {@link #getSubmitPriority()} value determinesthe order in which all steps will be submitted for execution. The default submit priority is 5.
Use the {@link FlowListener} to receive any events on the life-cycle of the Flow as it executes. Any{@link Tap} instances owned by the Flow also implementing FlowListener will automatically be added to theset of listeners. @see FlowListener @see cascading.flow.FlowConnector

    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );
    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter );


    Pipe splice = new HashJoin( pipeLower, new Fields( 0 ), pipeUpper, new Fields( 0 ), Fields.size( 4 ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );


    flow.complete();


    validateLength( flow, 5 );


    List<Tuple> values = getSinkAsList( flow );

View Full Code Here

    pipeUpper = new Each( pipeUpper, new Fields( "num" ), new RegexFilter( "^fobar" ) ); // intentionally filtering all
    pipeUpper = new GroupBy( pipeUpper, new Fields( "num" ) );


    Pipe splice = new HashJoin( pipeLower, new Fields( "num" ), pipeUpper, new Fields( "num" ), Fields.size( 4 ), new OuterJoin() );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );


    flow.complete();


    validateLength( flow, 5 );


    List<Tuple> values = getSinkAsList( flow );

View Full Code Here

    {
    getPlatform().copyFromLocal( inputFileIps );


    String path = "perpetual";


    Flow first = firstFlow( path );
    Flow second = secondFlow( first.getSink(), path );
    Flow third = thirdFlow( second.getSink(), path );
    Flow fourth = fourthFlow( third.getSink(), path );


    ProcessChain chain = new ProcessChain( true, fourth, second, first, third );


    chain.start();

View Full Code Here

    {
    getPlatform().copyFromLocal( inputFileIps );


    String path = "perpetualcascade";


    Flow first = firstFlow( path );
    Flow second = secondFlow( first.getSink(), path );
    Flow third = thirdFlow( second.getSink(), path );
    Flow fourth = fourthFlow( third.getSink(), path );


    ProcessFlow firstProcess = new ProcessFlow( "first", first );
    ProcessFlow secondProcess = new ProcessFlow( "second", second );
    ProcessFlow thirdProcess = new ProcessFlow( "third", third );
    ProcessFlow fourthProcess = new ProcessFlow( "fourth", fourth );

View Full Code Here

    Pipe pipe = new Pipe( "failing mapper" );
    pipe = new Each( pipe, new Fields( "line" ), new FailFunction( new Fields( "ip" ) ), new Fields( "ip" ) );


    Tap sink = getPlatform().getTextFile( getOutputPath( "mapperfail" ), SinkMode.REPLACE );


    Flow flow = getPlatform().getFlowConnector().connect( "mapper fail test", source, sink, pipe );


    Cascade cascade = new CascadeConnector().connect( flow );
    assertNull( cascade.getCascadeStats().getThrowable() );


    try

View Full Code Here

    pipe = new GroupBy( pipe, new Fields( "ip" ) );
    pipe = new Every( pipe, new TestFailAggregator( new Fields( "count" ), 1 ) );


    Tap sink = getPlatform().getTextFile( getOutputPath( "reducerfail" ), SinkMode.REPLACE );


    Flow flow = getPlatform().getFlowConnector().connect( "reducer fail test", source, sink, pipe );


    Cascade cascade = new CascadeConnector().connect( flow );
    assertNull( cascade.getCascadeStats().getThrowable() );
    try
      {

View Full Code Here

    Pipe left = new Each( new Pipe( "left", pipe ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( new Pipe( "right", pipe ), new Fields( "line" ), new RegexFilter( ".*102.*" ) );


    Pipe merged = new GroupBy( "merged", Pipe.pipes( left, right ), new Fields( "line" ) );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, merged );


    flow.complete();


    validateLength( flow, 3 );
    }

View Full Code Here

    Pipe pipeLower1 = new Pipe( "lower1" );


    Pipe left = new GroupBy( "output1", pipeLower1, new Fields( 0 ) );
    Pipe right = new GroupBy( "output2", left, new Fields( 0 ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, Pipe.pipes( left, right ) );


//    flow.writeDOT( "spit.dot" );


    flow.complete();


    validateLength( flow, 10, "output1" );
    validateLength( flow, 10, "output2" );


    assertEquals( 10, asSet( flow, sink1 ).size() );

View Full Code Here

    Pipe right = new Each( new Pipe( "right", pipe ), new Fields( "ip" ), new RegexFilter( ".*102.*" ) );


    Map sources = Cascades.tapsMap( "split", source );
    Map sinks = Cascades.tapsMap( Pipe.pipes( left, right ), Tap.taps( sink1, sink2 ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, left, right );


    flow.complete();


    validateLength( flow, 1, "left" );
    validateLength( flow, 1, "right" );
    }

View Full Code Here


    Pipe pipe = new Each( new Pipe( "concat" ), new Fields( "line" ), splitter );


    Pipe splice = new GroupBy( pipe, new Fields( "num" ) );


    Flow countFlow = getPlatform().getFlowConnector().connect( source, sink, splice );


    countFlow.complete();


    validateLength( countFlow, 10, null );
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of cascading.flow.Flow

bixo.examples.crawl.DemoCrawlTool

bixo.examples.crawl.DemoCrawlWorkflow

bixo.examples.crawl.DemoCrawlWorkflowLRTest

bixo.examples.crawl.LatestUrlDatumBufferTest

bixo.examples.crawl.UrlImporter

bixo.examples.webmining.DemoWebMiningTool

bixo.examples.webmining.DemoWebMiningWorkflow

bixo.examples.webmining.DemoWebMiningWorkflowTest

bixo.fetcher.FetcherTest

bixo.pipes.AbstractFetchPipeTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.