Examples of cascading.flow.Flow

cascading.flow.Flow
A Flow is a logical unit of work declared by an assembly of {@link cascading.pipe.Pipe} instances connected to sourceand sink {@link Tap} instances.
A Flow is then executed to push the incoming source data through the assembly into one or more sinks.
A Flow sub-class instance may not be instantiated directly in most cases, see sub-classes of {@link FlowConnector} classfor supported platforms.
Note that {@link cascading.pipe.Pipe} assemblies can be reused in multiple Flow instances. They maintainno state regarding the Flow execution. Subsequently, {@link cascading.pipe.Pipe} assemblies can be givenparameters through its calling Flow so they can be built in a generic fashion.
When a Flow is created, an optimized internal representation is created that is then executed on the underlying execution platform. This is typically done by creating one or more {@link FlowStep} instances.
Flows are submitted in order of dependency when used with a {@link cascading.cascade.Cascade}. If two or more steps do not share the same dependencies and all can be scheduled simultaneously, the {@link #getSubmitPriority()} value determinesthe order in which all steps will be submitted for execution. The default submit priority is 5.
Use the {@link FlowListener} to receive any events on the life-cycle of the Flow as it executes. Any{@link Tap} instances owned by the Flow also implementing FlowListener will automatically be added to theset of listeners. @see FlowListener @see cascading.flow.FlowConnector

     .addSource( stopPipe, stopTap )
     .addTailSink( tfidfPipe, tfidfTap )
     .addTailSink( wcPipe, wcTap );


    //run ambrose and cascading
    Flow tfidfFlow = flowConnector.connect( flowDef );
    EmbeddedAmbroseCascadingNotifier server = new EmbeddedAmbroseCascadingNotifier();
    tfidfFlow.addListener(server);
    tfidfFlow.addStepListener(server);
    tfidfFlow.complete();
    }

View Full Code Here

        new SerializingHBaseScheme(keyFields, valueFields,
                                       new Class<?>[]{String.class, String.class},
          false, SerializingHBaseScheme.Direction.FOR_WRITE),
        SinkMode.REPLACE);


    Flow flow = new FlowConnector(PROPERTIES).connect(source, hBaseTap, pipe);
    flow.complete();


    mHelper.expectResult(flow, expected);
  }

View Full Code Here

      Tap sink = new Hfs(new SequenceFile(output), outputPaths[i]);
      sinks.put(sinkName, sink);
      i++;
    }


    Flow f = mFlowConnector.connect(sources, sinks, tails);
    f.complete();
    return f;
  }

View Full Code Here


public class CascadingUtils {
    public static void identityFlow(Tap source, Tap sink, Fields selectFields) {
        Pipe pipe = new Pipe("pipe");
        pipe = new Each(pipe, selectFields, new Identity());
        Flow flow = new HadoopFlowConnector().connect(source, sink, pipe);
        flow.complete();
    }

View Full Code Here


      flowDef.addAssemblyPlanner( pmmlPlanner );
      }


    // write a DOT file and run the flow
    Flow classifyFlow = flowConnector.connect( flowDef );
    classifyFlow.writeDOT( "dot/classify.dot" );
    classifyFlow.complete();
    }

View Full Code Here

      .setPMMLInput( new File( "data/iris.lm_p.xml" ) )
      .retainOnlyActiveIncomingFields();


    flowDef.addAssemblyPlanner( pmmlPlanner );


    Flow flow = new LocalFlowConnector().connect( flowDef );


    flow.complete();


    TupleEntryIterator iterator = resultsTap.openForRead( flow.getFlowProcess() );


    while( iterator.hasNext() )
      System.out.println( iterator.next() );


    iterator.close();

View Full Code Here

      .addSource( "head", source )
      .addSink( "tail", sink )
      .addTail( pipe )
      .addAssemblyPlanner( planner );


    Flow flow;


    try
      {
      flow = getPlatform().getFlowConnector().connect( flowDef );
      }
    catch( PlannerException exception )
      {
      exception.writeDOT( getFlowPlanPath() + "/plan.dot" );


      throw exception;
      }


    flow.writeDOT( getFlowPlanPath() + "/plan.dot" );


    flow.complete();


    LOG.debug( "source = {}", source.getSourceFields().printVerbose() );
    LOG.debug( "sink   = {}", sink.getSinkFields().printVerbose() );


    Fields sourceSelector = source.getSourceFields().subtract( trainingFields );

View Full Code Here

    FlowDef flowDef = FlowDef.flowDef()
      .addSource( "head", source )
      .addSink( "tail", sink )
      .addTail( pipe );


    Flow flow = getPlatform().getFlowConnector().connect( flowDef );


    flow.writeDOT( getFlowPlanPath() + "/plan.dot" );


    flow.complete();


    Fields sourceSelector = source.getSourceFields();
    Fields sinkSelector = sink.getSinkFields();


    LOG.debug( "source select = {}", sourceSelector.printVerbose() );

View Full Code Here

    String[] primaryKeys = {"num", "lower"};
    TableDesc tableDesc = new TableDesc( tableName, columnNames, columnDefs, primaryKeys );


    Tap replaceTap = new JDBCTap( url, driver, tableDesc, new JDBCScheme( columnNames ), SinkMode.REPLACE );


    Flow parseFlow = new FlowConnector( getProperties() ).connect( source, replaceTap, parsePipe );


    parseFlow.complete();


    verifySink( parseFlow, 13 );


    // READ DATA FROM TABLE INTO TEXT FILE


    // create flow to read from hbase and save to local file
    Tap sink = new Lfs( new TextLine(), "build/test/jdbc", SinkMode.REPLACE );


    Pipe copyPipe = new Each( "read", new Identity() );


    Flow copyFlow = new FlowConnector( getProperties() ).connect( replaceTap, sink, copyPipe );


    copyFlow.complete();


    verifySink( copyFlow, 13 );


    // READ DATA FROM TEXT FILE AND UPDATE TABLE


    JDBCScheme jdbcScheme = new JDBCScheme( columnNames, null, new String[]{"num", "lower"} );
    Tap updateTap = new JDBCTap( url, driver, tableDesc, jdbcScheme, SinkMode.APPEND );


    Flow updateFlow = new FlowConnector( getProperties() ).connect( sink, updateTap, parsePipe );


    updateFlow.complete();


    verifySink( updateFlow, 13 );


    // READ DATA FROM TABLE INTO TEXT FILE, USING CUSTOM QUERY


    Tap sourceTap = new JDBCTap( url, driver, new JDBCScheme( columnNames, "select num, lower, upper from testingtable as testingtable", "select count(*) from testingtable" ) );


    Pipe readPipe = new Each( "read", new Identity() );


    Flow readFlow = new FlowConnector( getProperties() ).connect( sourceTap, sink, readPipe );


    readFlow.complete();


    verifySink( readFlow, 13 );
    }

View Full Code Here

    String[] primaryKeys = {"db_num", "db_lower"};
    TableDesc tableDesc = new TableDesc( tableName, columnNames, columnDefs, primaryKeys );


    Tap replaceTap = new JDBCTap( url, driver, tableDesc, new JDBCScheme( columnFields, columnNames ), SinkMode.REPLACE );


    Flow parseFlow = new FlowConnector( getProperties() ).connect( source, replaceTap, parsePipe );


    parseFlow.complete();


    verifySink( parseFlow, 13 );


    // READ DATA FROM TABLE INTO TEXT FILE


    // create flow to read from hbase and save to local file
    Tap sink = new Lfs( new TextLine(), "build/test/jdbc", SinkMode.REPLACE );


    Pipe copyPipe = new Each( "read", new Identity() );


    Flow copyFlow = new FlowConnector( getProperties() ).connect( replaceTap, sink, copyPipe );


    copyFlow.complete();


    verifySink( copyFlow, 13 );


    // READ DATA FROM TEXT FILE AND UPDATE TABLE


    Fields updateByFields = new Fields( "num", "lower" );
    String[] updateBy = {"db_num", "db_lower"};
    JDBCScheme jdbcScheme = new JDBCScheme( columnFields, columnNames, null, updateByFields, updateBy );
    Tap updateTap = new JDBCTap( url, driver, tableDesc, jdbcScheme, SinkMode.APPEND );


    Flow updateFlow = new FlowConnector( getProperties() ).connect( sink, updateTap, parsePipe );


    updateFlow.complete();


    verifySink( updateFlow, 13 );


    // READ DATA FROM TABLE INTO TEXT FILE, USING CUSTOM QUERY


    Tap sourceTap = new JDBCTap( url, driver, new JDBCScheme( columnFields, columnNames, "select db_num, db_lower, db_upper from testingtablealias as testingtablealias", "select count(*) from testingtablealias" ) );


    Pipe readPipe = new Each( "read", new Identity() );


    Flow readFlow = new FlowConnector( getProperties() ).connect( sourceTap, sink, readPipe );


    readFlow.complete();


    verifySink( readFlow, 13 );
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of cascading.flow.Flow

bixo.examples.crawl.DemoCrawlTool

bixo.examples.crawl.DemoCrawlWorkflow

bixo.examples.crawl.DemoCrawlWorkflowLRTest

bixo.examples.crawl.LatestUrlDatumBufferTest

bixo.examples.crawl.UrlImporter

bixo.examples.webmining.DemoWebMiningTool

bixo.examples.webmining.DemoWebMiningWorkflow

bixo.examples.webmining.DemoWebMiningWorkflowTest

bixo.fetcher.FetcherTest

bixo.pipes.AbstractFetchPipeTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.