Examples of cascading.flow.Flow

cascading.flow.Flow
A Flow is a logical unit of work declared by an assembly of {@link cascading.pipe.Pipe} instances connected to sourceand sink {@link Tap} instances.
A Flow is then executed to push the incoming source data through the assembly into one or more sinks.
A Flow sub-class instance may not be instantiated directly in most cases, see sub-classes of {@link FlowConnector} classfor supported platforms.
Note that {@link cascading.pipe.Pipe} assemblies can be reused in multiple Flow instances. They maintainno state regarding the Flow execution. Subsequently, {@link cascading.pipe.Pipe} assemblies can be givenparameters through its calling Flow so they can be built in a generic fashion.
When a Flow is created, an optimized internal representation is created that is then executed on the underlying execution platform. This is typically done by creating one or more {@link FlowStep} instances.
Flows are submitted in order of dependency when used with a {@link cascading.cascade.Cascade}. If two or more steps do not share the same dependencies and all can be scheduled simultaneously, the {@link #getSubmitPriority()} value determinesthe order in which all steps will be submitted for execution. The default submit priority is 5.
Use the {@link FlowListener} to receive any events on the life-cycle of the Flow as it executes. Any{@link Tap} instances owned by the Flow also implementing FlowListener will automatically be added to theset of listeners. @see FlowListener @see cascading.flow.FlowConnector


    Filter filter = new And( new RegexFilter( "^68.*$" ), new RegexFilter( "^1000.*$" ) );


    pipe = new Each( pipe, new Fields( "line" ), filter );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );


    flow.complete();


    validateLength( flow, 3 );
    }

View Full Code Here


    pipe = new GroupBy( pipe, new Fields( "value" ) );


    pipe = new Every( pipe, new Count(), new Fields( "value", "count" ) );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );


    flow.complete();


    validateLength( flow, 1, null );
    }

View Full Code Here


    pipe = new GroupBy( pipe, new Fields( "value" ) );


    pipe = new Every( pipe, new Count(), new Fields( "value", "count" ) );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );


    flow.complete();


    validateLength( flow, 0, null );
    }

View Full Code Here

    {
    copyFromLocal( InputData.inputFileNums20 );
    Tap source = getPlatform().getTextFile( InputData.inputFileNums20 );
    Tap sink = getPlatform().getTextFile( getOutputPath( key ), SinkMode.REPLACE );


    Flow flow = null;


    try
      {
      flow = getPlatform().getFlowConnector().connect( source, sink, pipe );


      if( isWriteDOT() )
        flow.writeDOT( getName() + ".dot" ); // use display name


      flow.complete();


      if( isError() )
        fail( "did not throw asserted error" );
      }
    catch( Exception exception )
      {
      if( isError() )
        return;
      else
        throw exception;
      }


    if( resultLength != -1 )
      validateLength( flow, resultLength );


    TupleEntryIterator iterator = flow.openSink();
    Object result = iterator.next().getObject( 1 );


    if( resultTuple != null )
      assertEquals( "not equal: ", resultTuple.toString(), result );
    else if( resultTuple == null )

View Full Code Here


    Map sinks = new HashMap();
    sinks.put( "left", sink1 );
    sinks.put( "right", sink2 );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, left, right );


    flow.complete();


    validateLength( flow, 1, "left" );
    validateLength( flow, 2, "right" );
    }

View Full Code Here


    Map sinks = new HashMap();
    sinks.put( "left", sink1 );
    sinks.put( "right", sink2 );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, left, right );


    flow.complete();


    validateLength( flow, 1, "left" );
    validateLength( flow, 2, "right" );
    }

View Full Code Here

    pipe = new Each( pipe, new Counter( TestEnum.SECOND ) );


    Tap sink1 = getPlatform().getTextFile( getOutputPath( "flowstats1" ), SinkMode.REPLACE );
    Tap sink2 = getPlatform().getTextFile( getOutputPath( "flowstats2" ), SinkMode.REPLACE );


    Flow flow1 = getPlatform().getFlowConnector().connect( "stats1 test", source, sink1, pipe );
    Flow flow2 = getPlatform().getFlowConnector().connect( "stats2 test", source, sink2, pipe );


    Cascade cascade = new CascadeConnector().connect( flow1, flow2 );


    cascade.complete();


    CascadeStats cascadeStats = cascade.getCascadeStats();


    assertNotNull( cascadeStats.getID() );


    // unsure why this has changed
//    if( getPlatform() instanceof HadoopPlatform )
//      {
//      Collection<String> counterGroups = cascadeStats.getCounterGroups();
//      assertEquals( getPlatform().isUseCluster() ? 5 : 4, counterGroups.size() );
//      }


    assertEquals( 1, cascadeStats.getCounterGroupsMatching( "cascading\\.stats\\..*" ).size() );
    assertEquals( 2, cascadeStats.getCountersFor( TestEnum.class.getName() ).size() );
    assertEquals( 2, cascadeStats.getCountersFor( TestEnum.class ).size() );
    assertEquals( 40, cascadeStats.getCounterValue( TestEnum.FIRST ) );
    assertEquals( 20, cascadeStats.getCounterValue( TestEnum.SECOND ) );


    // should not throw npe
    assertEquals( 0, cascadeStats.getCounterValue( TestEnum.THIRD ) );
    assertEquals( 0, cascadeStats.getCounterValue( "FOO", "BAR" ) );


    FlowStats flowStats1 = flow1.getFlowStats();


    assertNotNull( flowStats1.getID() );


    assertEquals( 20, flowStats1.getCounterValue( TestEnum.FIRST ) );
    assertEquals( 10, flowStats1.getCounterValue( TestEnum.SECOND ) );


    assertEquals( 10, flowStats1.getFlowStepStats().get( 0 ).getCounterValue( TestEnum.FIRST ) );
    assertEquals( 10, flowStats1.getFlowStepStats().get( 1 ).getCounterValue( TestEnum.FIRST ) );


    // should not throw npe
    assertEquals( 0, flowStats1.getCounterValue( TestEnum.THIRD ) );
    assertEquals( 0, flowStats1.getCounterValue( "FOO", "BAR" ) );


    FlowStats flowStats2 = flow2.getFlowStats();


    assertNotNull( flowStats2.getID() );


    assertEquals( 20, flowStats2.getCounterValue( TestEnum.FIRST ) );
    assertEquals( 10, flowStats2.getCounterValue( TestEnum.SECOND ) );

View Full Code Here

    Pipe pipeLower = new Each( "lhs", new Fields( "line" ), new RegexSplitter( new Fields( "numLHS", "charLHS" ), " " ) );
    Pipe pipeUpper = new Each( "rhs", new Fields( "line" ), new RegexSplitter( new Fields( "numRHS", "charRHS" ), " " ) );


    Pipe cross = new HashJoin( pipeLower, new Fields( "numLHS" ), pipeUpper, new Fields( "numRHS" ), new InnerJoin() );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, cross );


    flow.complete();


    validateLength( flow, 37, null );


    List<Tuple> values = getSinkAsList( flow );

View Full Code Here


    Pipe splice = new HashJoin( pipeLower, new Fields( "num" ), pipeUpper, new Fields( "num" ), Fields.size( 4 ) );


    Map<Object, Object> properties = getProperties();


    Flow flow = getPlatform().getFlowConnector( properties ).connect( sources, sink, splice );


    flow.complete();


    validateLength( flow, 5 );


    List<Tuple> values = getSinkAsList( flow );

View Full Code Here


//    splice = new Each( splice, new Debug( true ) );
    splice = new Pipe( "splice", splice );
    splice = new Pipe( "tail", splice );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );


    flow.complete();


    validateLength( flow, 5 );


    List<Tuple> values = getSinkAsList( flow );

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of cascading.flow.Flow

bixo.examples.crawl.DemoCrawlTool

bixo.examples.crawl.DemoCrawlWorkflow

bixo.examples.crawl.DemoCrawlWorkflowLRTest

bixo.examples.crawl.LatestUrlDatumBufferTest

bixo.examples.crawl.UrlImporter

bixo.examples.webmining.DemoWebMiningTool

bixo.examples.webmining.DemoWebMiningWorkflow

bixo.examples.webmining.DemoWebMiningWorkflowTest

bixo.fetcher.FetcherTest

bixo.pipes.AbstractFetchPipeTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.