Examples of addInputToGroup()

Examples of org.apache.flink.runtime.operators.util.TaskConfig.addInputToGroup()

207
208
209
210
211
212
213
214
215
216
217
//    intermediateConfig.setDriver(RepeatableHashjoinMatchDriverWithCachedBuildside.class);
    intermediateConfig.setDriver(BuildSecondCachedMatchDriver.class);
    intermediateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
    intermediateConfig.setRelativeMemoryDriver((double)matchMemory/totalMemoryConsumption);
    intermediateConfig.addInputToGroup(0);
    intermediateConfig.addInputToGroup(1);
    intermediateConfig.setInputSerializer(vertexWithRankAndDanglingSerializer, 0);
    intermediateConfig.setInputSerializer(vertexWithAdjacencyListSerializer, 1);
    intermediateConfig.setDriverComparator(vertexWithRankAndDanglingComparator, 0);
    intermediateConfig.setDriverComparator(vertexWithAdjacencyListComparator, 1);
    intermediateConfig.setDriverPairComparator(matchComparator);
View Full Code Here

Examples of org.apache.flink.runtime.operators.util.TaskConfig.addInputToGroup()

225
226
227
228
229
230
231
232
233
234
235
    intermediateConfig.setStubParameter("compensation.failingIteration", String.valueOf(failingIteration));
    intermediateConfig.setStubParameter("compensation.messageLoss", String.valueOf(messageLoss));
   
    // the combiner and the output
    TaskConfig combinerConfig = new TaskConfig(new Configuration());
    combinerConfig.addInputToGroup(0);
    combinerConfig.setInputSerializer(vertexWithRankSerializer, 0);
    combinerConfig.setDriverStrategy(DriverStrategy.SORTED_GROUP_COMBINE);
    combinerConfig.setDriverComparator(vertexWithRankComparator, 0);
    combinerConfig.setDriverComparator(vertexWithRankComparator, 1);
    combinerConfig.setRelativeMemoryDriver((double)coGroupSortMemory/totalMemoryConsumption);
View Full Code Here

Examples of org.apache.flink.runtime.operators.util.TaskConfig.addInputToGroup()

245
246
247
248
249
250
251
252
253
254
255
   
    // --------------- the output ---------------------

    OutputFormatVertex output = JobGraphUtils.createFileOutput(jobGraph, "FinalOutput", degreeOfParallelism);
    TaskConfig outputConfig = new TaskConfig(output.getConfiguration());
    outputConfig.addInputToGroup(0);
    outputConfig.setInputSerializer(recSerializer, 0);
    outputConfig.setStubWrapper(new UserCodeClassWrapper<PageWithRankOutFormat>(PageWithRankOutFormat.class));
    outputConfig.setStubParameter(FileOutputFormat.FILE_PARAMETER_KEY, outputPath);
   
    // --------------- the auxiliaries ---------------------
View Full Code Here

Examples of org.apache.flink.runtime.operators.util.TaskConfig.addInputToGroup()

248
249
250
251
252
253
254
255
256
257
258
    tailConfig.setIsWorksetUpdate();
   
    // inputs and driver
    tailConfig.setDriver(CoGroupDriver.class);
    tailConfig.setDriverStrategy(DriverStrategy.CO_GROUP);
    tailConfig.addInputToGroup(0);
    tailConfig.addInputToGroup(1);
    tailConfig.setInputSerializer(vertexWithRankAndDanglingSerializer, 0);
    tailConfig.setInputSerializer(vertexWithRankSerializer, 1);
    tailConfig.setDriverComparator(vertexWithRankAndDanglingComparator, 0);
    tailConfig.setDriverComparator(vertexWithRankComparator, 1);
View Full Code Here

Examples of org.apache.flink.runtime.operators.util.TaskConfig.addInputToGroup()

249
250
251
252
253
254
255
256
257
258
259
   
    // inputs and driver
    tailConfig.setDriver(CoGroupDriver.class);
    tailConfig.setDriverStrategy(DriverStrategy.CO_GROUP);
    tailConfig.addInputToGroup(0);
    tailConfig.addInputToGroup(1);
    tailConfig.setInputSerializer(vertexWithRankAndDanglingSerializer, 0);
    tailConfig.setInputSerializer(vertexWithRankSerializer, 1);
    tailConfig.setDriverComparator(vertexWithRankAndDanglingComparator, 0);
    tailConfig.setDriverComparator(vertexWithRankComparator, 1);
    tailConfig.setDriverPairComparator(coGroupComparator);
View Full Code Here

Examples of org.apache.flink.runtime.operators.util.TaskConfig.addInputToGroup()

279
280
281
282
283
284
285
286
287
288
289
   
    // --------------- the output ---------------------

    OutputFormatVertex output = JobGraphUtils.createFileOutput(jobGraph, "FinalOutput", degreeOfParallelism);
    TaskConfig outputConfig = new TaskConfig(output.getConfiguration());
    outputConfig.addInputToGroup(0);
    outputConfig.setInputSerializer(vertexWithRankAndDanglingSerializer, 0);
    outputConfig.setStubWrapper(new UserCodeClassWrapper<CustomPageWithRankOutFormat>(CustomPageWithRankOutFormat.class));
    outputConfig.setStubParameter(FileOutputFormat.FILE_PARAMETER_KEY, outputPath);
   
    // --------------- the auxiliaries ---------------------
View Full Code Here

Examples of org.apache.flink.runtime.operators.util.TaskConfig.addInputToGroup()

246
247
248
249
250
251
252
253
254
255
256
      headConfig.setInputLocalStrategy(1, LocalStrategy.NONE);
      headConfig.setInputCached(1, true);
      headConfig.setRelativeInputMaterializationMemory(1, MEM_FRAC_PER_CONSUMER);

      // initial solution set input
      headConfig.addInputToGroup(2);
      headConfig.setInputSerializer(serializer, 2);
      headConfig.setInputComparator(comparator, 2);
      headConfig.setInputLocalStrategy(2, LocalStrategy.NONE);
      headConfig.setIterationHeadSolutionSetInputIndex(2);
View Full Code Here

Examples of org.apache.flink.runtime.operators.util.TaskConfig.addInputToGroup()

301
302
303
304
305
306
307
308
309
310
311
      "Find Min Component-ID", jobGraph, numSubTasks);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());
    {
      intermediateConfig.setIterationId(ITERATION_ID);

      intermediateConfig.addInputToGroup(0);
      intermediateConfig.setInputSerializer(serializer, 0);
      intermediateConfig.setInputComparator(comparator, 0);
      intermediateConfig.setInputLocalStrategy(0, LocalStrategy.SORT);
      intermediateConfig.setRelativeMemoryInput(0, MEM_FRAC_PER_CONSUMER);
      intermediateConfig.setFilehandlesInput(0, 64);
View Full Code Here

Examples of org.apache.flink.runtime.operators.util.TaskConfig.addInputToGroup()

273
274
275
276
277
278
279
280
281
282
283
      taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      taskConfig.setOutputSerializer(serializer);
      taskConfig.setDriver(CollectorMapDriver.class);
      taskConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);

      taskConfig.addInputToGroup(0);
      taskConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
      taskConfig.setInputSerializer(serializer, 0);

      taskConfig.setBroadcastInputName("models", 0);
      taskConfig.addBroadcastInputToGroup(0);
View Full Code Here

Examples of org.apache.flink.runtime.operators.util.TaskConfig.addInputToGroup()

290
291
292
293
294
295
296
297
298
299
300
  private static OutputFormatVertex createOutput(JobGraph jobGraph, String resultPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
    OutputFormatVertex output = JobGraphUtils.createFileOutput(jobGraph, "Output", numSubTasks);

    {
      TaskConfig taskConfig = new TaskConfig(output.getConfiguration());
      taskConfig.addInputToGroup(0);
      taskConfig.setInputSerializer(serializer, 0);

      @SuppressWarnings("unchecked")
      CsvOutputFormat outFormat = new CsvOutputFormat("\n", " ", LongValue.class, LongValue.class, LongValue.class);
      outFormat.setOutputFilePath(new Path(resultPath));
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.