Package cascading.flow

Examples of cascading.flow.Flow.complete()

187
188
189
190
191
192
193
194
195
196
197
      sinks.put(sinkName, sink);
      i++;
    }

    Flow f = mFlowConnector.connect(sources, sinks, tails);
    f.complete();
    return f;
  }

  /**
   * Fails Junit if tuple entry iterator entries do not match the expected list.
View Full Code Here
18
19
20
21
22
23
24
25
26
27
28
public class CascadingUtils {
    public static void identityFlow(Tap source, Tap sink, Fields selectFields) {
        Pipe pipe = new Pipe("pipe");
        pipe = new Each(pipe, selectFields, new Identity());
        Flow flow = new HadoopFlowConnector().connect(source, sink, pipe);
        flow.complete();
    }

    // Mark the output dir of the job for which the context is passed.
    public static void markSuccessfulOutputDir(Path path, JobConf conf) throws IOException {
        FileSystem fs = FileSystem.get(conf);
View Full Code Here
79
80
81
82
83
84
85
86
      }

    // write a DOT file and run the flow
    Flow classifyFlow = flowConnector.connect( flowDef );
    classifyFlow.writeDOT( "dot/classify.dot" );
    classifyFlow.complete();
    }
  }
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
    flowDef.addAssemblyPlanner( pmmlPlanner );

    Flow flow = new LocalFlowConnector().connect( flowDef );

    flow.complete();

    TupleEntryIterator iterator = resultsTap.openForRead( flow.getFlowProcess() );

    while( iterator.hasNext() )
      System.out.println( iterator.next() );
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
      throw exception;
      }

    flow.writeDOT( getFlowPlanPath() + "/plan.dot" );

    flow.complete();

    LOG.debug( "source = {}", source.getSourceFields().printVerbose() );
    LOG.debug( "sink   = {}", sink.getSinkFields().printVerbose() );

    Fields sourceSelector = source.getSourceFields().subtract( trainingFields );
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
    Flow flow = getPlatform().getFlowConnector().connect( flowDef );

    flow.writeDOT( getFlowPlanPath() + "/plan.dot" );

    flow.complete();

    Fields sourceSelector = source.getSourceFields();
    Fields sinkSelector = sink.getSinkFields();

    LOG.debug( "source select = {}", sourceSelector.printVerbose() );
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
    Tap replaceTap = new JDBCTap( url, driver, tableDesc, new JDBCScheme( columnNames ), SinkMode.REPLACE );

    Flow parseFlow = new FlowConnector( getProperties() ).connect( source, replaceTap, parsePipe );

    parseFlow.complete();

    verifySink( parseFlow, 13 );

    // READ DATA FROM TABLE INTO TEXT FILE
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
    Pipe copyPipe = new Each( "read", new Identity() );

    Flow copyFlow = new FlowConnector( getProperties() ).connect( replaceTap, sink, copyPipe );

    copyFlow.complete();

    verifySink( copyFlow, 13 );

    // READ DATA FROM TEXT FILE AND UPDATE TABLE
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
    JDBCScheme jdbcScheme = new JDBCScheme( columnNames, null, new String[]{"num", "lower"} );
    Tap updateTap = new JDBCTap( url, driver, tableDesc, jdbcScheme, SinkMode.APPEND );

    Flow updateFlow = new FlowConnector( getProperties() ).connect( sink, updateTap, parsePipe );

    updateFlow.complete();

    verifySink( updateFlow, 13 );

    // READ DATA FROM TABLE INTO TEXT FILE, USING CUSTOM QUERY
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
    Pipe readPipe = new Each( "read", new Identity() );

    Flow readFlow = new FlowConnector( getProperties() ).connect( sourceTap, sink, readPipe );

    readFlow.complete();

    verifySink( readFlow, 13 );
    }

  public void testJDBCAliased() throws IOException
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.