Examples of cascading.flow.Flow

cascading.flow.Flow
A Flow is a logical unit of work declared by an assembly of {@link cascading.pipe.Pipe} instances connected to sourceand sink {@link Tap} instances.
A Flow is then executed to push the incoming source data through the assembly into one or more sinks.
A Flow sub-class instance may not be instantiated directly in most cases, see sub-classes of {@link FlowConnector} classfor supported platforms.
Note that {@link cascading.pipe.Pipe} assemblies can be reused in multiple Flow instances. They maintainno state regarding the Flow execution. Subsequently, {@link cascading.pipe.Pipe} assemblies can be givenparameters through its calling Flow so they can be built in a generic fashion.
When a Flow is created, an optimized internal representation is created that is then executed on the underlying execution platform. This is typically done by creating one or more {@link FlowStep} instances.
Flows are submitted in order of dependency when used with a {@link cascading.cascade.Cascade}. If two or more steps do not share the same dependencies and all can be scheduled simultaneously, the {@link #getSubmitPriority()} value determinesthe order in which all steps will be submitted for execution. The default submit priority is 5.
Use the {@link FlowListener} to receive any events on the life-cycle of the Flow as it executes. Any{@link Tap} instances owned by the Flow also implementing FlowListener will automatically be added to theset of listeners. @see FlowListener @see cascading.flow.FlowConnector


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );


    Pipe cogroup = new CoGroup( pipeLower, new Fields( "num" ), 1, new Fields( "num1", "char1", "num2", "char2" ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, cogroup );


    flow.complete();


    validateLength( flow, 5, null );


    List<Tuple> actual = getSinkAsList( flow );

View Full Code Here


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );


    Pipe cogroup = new CoGroup( pipeLower, new Fields( "num" ), pipeLower, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2" ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, cogroup );


    flow.complete();


    validateLength( flow, 5, null );


    List<Tuple> actual = getSinkAsList( flow );

View Full Code Here

    Pipe lhs = new Pipe( "lhs", pipe );
    Pipe rhs = new Pipe( "rhs", pipe );


    Pipe cogroup = new CoGroup( lhs, new Fields( "num" ), rhs, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2" ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, cogroup );


    flow.complete();


    validateLength( flow, 5, null );


    List<Tuple> actual = getSinkAsList( flow );

View Full Code Here


    splice1 = new Each( splice1, new Identity() );


    Pipe splice2 = new CoGroup( splice1, new Fields( "num1" ), pipeUpper2, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2", "num3", "char3" ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice2 );


    flow.complete();


    validateLength( flow, 5, null );


    List<Tuple> actual = getSinkAsList( flow );

View Full Code Here

    Map<Object, Object> properties = getPlatform().getProperties();


    if( getPlatform().isMapReduce() )
      FlowConnectorProps.setIntermediateSchemeClass( properties, schemeClass );


    Flow flow = getPlatform().getFlowConnector( properties ).connect( "cogroupopt", sources, sink, splice2 );


    if( getPlatform().isMapReduce() )
      assertEquals( "wrong number of steps", 2, flow.getFlowSteps().size() );


    flow.complete();


    validateLength( flow, 10 );


    List<Tuple> actual = getSinkAsList( flow );

View Full Code Here


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitterUpper );


    Pipe cogroup = new CoGroup( offsetLower, new Fields( "numA" ), pipeLower, new Fields( "numB" ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, cogroup );


    flow.complete();


    validateLength( flow, 5 );


    List<Tuple> actual = getSinkAsList( flow );

View Full Code Here


    Pipe splice = new CoGroup( pipeLower, Fields.NONE, pipeUpper, Fields.NONE, Fields.size( 4 ) );


    Map<Object, Object> properties = getProperties();


    Flow flow = getPlatform().getFlowConnector( properties ).connect( sources, sink, splice );


    flow.complete();


    validateLength( flow, 25 );


    List<Tuple> values = getSinkAsList( flow );

View Full Code Here

    Map<String, Tap> sources = Cascades.tapsMap( heads, Tap.taps( source, source ) );


    Pipe[] tails = Pipe.pipes( inner, outer, left, right );
    Map<String, Tap> sinks = Cascades.tapsMap( tails, Tap.taps( innerSink, outerSink, leftSink, rightSink ) );


    Flow flow = getPlatform().getFlowConnector().connect( "multi-joins", sources, sinks, tails );


    flow.complete();


    validateLength( flow.openTapForRead( innerSink ), 74 );
    validateLength( flow.openTapForRead( outerSink ), 84 );
    validateLength( flow.openTapForRead( leftSink ), 74 );
    validateLength( flow.openTapForRead( rightSink ), 84 );
    }

View Full Code Here


    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );


    Tap sink = getPlatform().getTextFile( getOutputPath( "simple" ), SinkMode.REPLACE );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );


    flow.complete();


    validateLength( flow.openSource(), 10 ); // validate source, this once, as a sanity check
    validateLength( flow, 8, null );
    }

View Full Code Here

    pipe = new Every( pipe, new Count( new Fields( "count3" ) ) );
    pipe = new Every( pipe, new Count( new Fields( "count4" ) ) );


    Tap sink = getPlatform().getTabDelimitedFile( Fields.ALL, getOutputPath( "simplechain" ), SinkMode.REPLACE );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );


    flow.complete();


    validateLength( flow, 8, 5 );
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of cascading.flow.Flow

bixo.examples.crawl.DemoCrawlTool

bixo.examples.crawl.DemoCrawlWorkflow

bixo.examples.crawl.DemoCrawlWorkflowLRTest

bixo.examples.crawl.LatestUrlDatumBufferTest

bixo.examples.crawl.UrlImporter

bixo.examples.webmining.DemoWebMiningTool

bixo.examples.webmining.DemoWebMiningWorkflow

bixo.examples.webmining.DemoWebMiningWorkflowTest

bixo.fetcher.FetcherTest

bixo.pipes.AbstractFetchPipeTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.