Package org.apache.flink.runtime.operators.util

Examples of org.apache.flink.runtime.operators.util.TaskConfig.addInputToGroup()

301
302
303
304
305
306
307
308
309
310
311
      "Find Min Component-ID", jobGraph, numSubTasks);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());
    {
      intermediateConfig.setIterationId(ITERATION_ID);

      intermediateConfig.addInputToGroup(0);
      intermediateConfig.setInputSerializer(serializer, 0);
      intermediateConfig.setInputComparator(comparator, 0);
      intermediateConfig.setInputLocalStrategy(0, LocalStrategy.SORT);
      intermediateConfig.setRelativeMemoryInput(0, MEM_FRAC_PER_CONSUMER);
      intermediateConfig.setFilehandlesInput(0, 64);
View Full Code Here
273
274
275
276
277
278
279
280
281
282
283
      taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      taskConfig.setOutputSerializer(serializer);
      taskConfig.setDriver(CollectorMapDriver.class);
      taskConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);

      taskConfig.addInputToGroup(0);
      taskConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
      taskConfig.setInputSerializer(serializer, 0);

      taskConfig.setBroadcastInputName("models", 0);
      taskConfig.addBroadcastInputToGroup(0);
View Full Code Here
290
291
292
293
294
295
296
297
298
299
300
  private static OutputFormatVertex createOutput(JobGraph jobGraph, String resultPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
    OutputFormatVertex output = JobGraphUtils.createFileOutput(jobGraph, "Output", numSubTasks);

    {
      TaskConfig taskConfig = new TaskConfig(output.getConfiguration());
      taskConfig.addInputToGroup(0);
      taskConfig.setInputSerializer(serializer, 0);

      @SuppressWarnings("unchecked")
      CsvOutputFormat outFormat = new CsvOutputFormat("\n", " ", LongValue.class, LongValue.class, LongValue.class);
      outFormat.setOutputFilePath(new Path(resultPath));
View Full Code Here
328
329
330
331
332
333
334
335
336
337
338
      TypeSerializerFactory<?> serializer) {
    OutputFormatVertex output = JobGraphUtils.createFileOutput(jobGraph, "Final Output", numSubTasks);
    TaskConfig outputConfig = new TaskConfig(output.getConfiguration());
    {

      outputConfig.addInputToGroup(0);
      outputConfig.setInputSerializer(serializer, 0);

      outputConfig.setStubWrapper(new UserCodeClassWrapper<CsvOutputFormat>(CsvOutputFormat.class));
      outputConfig.setStubParameter(FileOutputFormat.FILE_PARAMETER_KEY, resultPath);
View Full Code Here
400
401
402
403
404
405
406
407
408
409
410
      tailConfig.setIsSolutionSetUpdate();
      tailConfig.setIsSolutionSetUpdateWithoutReprobe();

      // inputs and driver
      tailConfig.addInputToGroup(0);
      tailConfig.setInputSerializer(serializer, 0);

      // output
      tailConfig.setOutputSerializer(serializer);
View Full Code Here
482
483
484
485
486
487
488
489
490
491
492
    TaskConfig ssJoinIntermediateConfig = new TaskConfig(ssJoinIntermediate.getConfiguration());
    {
      ssJoinIntermediateConfig.setIterationId(ITERATION_ID);

      // inputs
      ssJoinIntermediateConfig.addInputToGroup(0);
      ssJoinIntermediateConfig.setInputSerializer(serializer, 0);

      // output
      ssJoinIntermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssJoinIntermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
   
    OutputFormatVertex output = JobGraphUtils.createFileOutput(jobGraph, "Output", numSubTasks);

    {
      TaskConfig taskConfig = new TaskConfig(output.getConfiguration());
      taskConfig.addInputToGroup(0);
      taskConfig.setInputSerializer(serializer, 0);

      PointOutFormat outFormat = new PointOutFormat();
      outFormat.setOutputFilePath(new Path(resultPath));
     
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    headConfig.setIterationId(ITERATION_ID);
   
    // initial input / partial solution
    headConfig.addInputToGroup(0);
    headConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
    headConfig.setInputSerializer(serializer, 0);
   
    // back channel / iterations
    headConfig.setRelativeBackChannelMemory(MEMORY_FRACTION_PER_CONSUMER);
View Full Code Here
200
201
202
203
204
205
206
207
208
209
210
    TaskConfig intermediateConfig = new TaskConfig(mapper.getConfiguration());
    intermediateConfig.setIterationId(ITERATION_ID);
   
    intermediateConfig.setDriver(CollectorMapDriver.class);
    intermediateConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
    intermediateConfig.addInputToGroup(0);
    intermediateConfig.setInputSerializer(inputSerializer, 0);
   
    intermediateConfig.setOutputSerializer(outputSerializer);
    intermediateConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
    intermediateConfig.setOutputComparator(outputComparator, 0);
View Full Code Here
513
514
515
516
517
518
519
520
521
522
523
      ssTailConfig.setIterationId(ITERATION_ID);
      ssTailConfig.setIsSolutionSetUpdate();
      ssTailConfig.setIsWorksetIteration();

      // inputs and driver
      ssTailConfig.addInputToGroup(0);
      ssTailConfig.setInputSerializer(serializer, 0);
      ssTailConfig.setInputAsynchronouslyMaterialized(0, true);
      ssTailConfig.setRelativeInputMaterializationMemory(0, MEM_FRAC_PER_CONSUMER);

      // output
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.