Package eu.stratosphere.pact.runtime.task.util

Examples of eu.stratosphere.pact.runtime.task.util.TaskConfig.addInputToGroup()

77
78
79
80
81
82
83
84
85
86
87
      // chained combine config
      {
        final TaskConfig combineConfig = new TaskConfig(new Configuration());
 
        // input
        combineConfig.addInputToGroup(0);
        combineConfig.setInputSerializer(serFact, 0);
       
        // output
        combineConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
        combineConfig.setOutputSerializer(serFact);
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
  }

  public void addInput(MutableObjectIterator<Record> input, int groupId) {
    this.mockEnv.addInput(input);
    TaskConfig conf = new TaskConfig(this.mockEnv.getTaskConfiguration());
    conf.addInputToGroup(groupId);
    conf.setInputSerializer(RecordSerializerFactory.get(), groupId);
  }

  public void addOutput(List<Record> output) {
    this.mockEnv.addOutput(output);
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
      // chained combine config
      {
        final TaskConfig combineConfig = new TaskConfig(new Configuration());
 
        // input
        combineConfig.addInputToGroup(0);
        combineConfig.setInputSerializer(serFact, 0);
       
        // output
        combineConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
        combineConfig.setOutputSerializer(serFact);
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
      degreeOfParallelism, numSubTasksPerInstance);
    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    headConfig.setIterationId(ITERATION_ID);
   
    // initial input / partial solution
    headConfig.addInputToGroup(0);
    headConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
    headConfig.setInputSerializer(recSerializer, 0);
    headConfig.setInputComparator(fieldZeroComparator, 0);
    headConfig.setInputLocalStrategy(0, LocalStrategy.SORT);
    headConfig.setMemoryInput(0, minorConsumer * JobGraphUtils.MEGABYTE);
 
View Full Code Here
180
181
182
183
184
185
186
187
188
189
190
    intermediateConfig.setIterationId(ITERATION_ID);
//    intermediateConfig.setDriver(RepeatableHashjoinMatchDriverWithCachedBuildside.class);
    intermediateConfig.setDriver(BuildSecondCachedMatchDriver.class);
    intermediateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
    intermediateConfig.setMemoryDriver(matchMemory * JobGraphUtils.MEGABYTE);
    intermediateConfig.addInputToGroup(0);
    intermediateConfig.addInputToGroup(1);
    intermediateConfig.setInputSerializer(recSerializer, 0);
    intermediateConfig.setInputSerializer(recSerializer, 1);
    intermediateConfig.setDriverComparator(fieldZeroComparator, 0);
    intermediateConfig.setDriverComparator(fieldZeroComparator, 1);
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
//    intermediateConfig.setDriver(RepeatableHashjoinMatchDriverWithCachedBuildside.class);
    intermediateConfig.setDriver(BuildSecondCachedMatchDriver.class);
    intermediateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
    intermediateConfig.setMemoryDriver(matchMemory * JobGraphUtils.MEGABYTE);
    intermediateConfig.addInputToGroup(0);
    intermediateConfig.addInputToGroup(1);
    intermediateConfig.setInputSerializer(recSerializer, 0);
    intermediateConfig.setInputSerializer(recSerializer, 1);
    intermediateConfig.setDriverComparator(fieldZeroComparator, 0);
    intermediateConfig.setDriverComparator(fieldZeroComparator, 1);
    intermediateConfig.setDriverPairComparator(pairComparatorFactory);
View Full Code Here
210
211
212
213
214
215
216
217
218
219
220
    // TODO we need to combine!
   
    // inputs and driver
    tailConfig.setDriver(CoGroupDriver.class);
    tailConfig.setDriverStrategy(DriverStrategy.CO_GROUP);
    tailConfig.addInputToGroup(0);
    tailConfig.addInputToGroup(1);
    tailConfig.setInputSerializer(recSerializer, 0);
    tailConfig.setInputSerializer(recSerializer, 1);
    tailConfig.setDriverComparator(fieldZeroComparator, 0);
    tailConfig.setDriverComparator(fieldZeroComparator, 1);
View Full Code Here
211
212
213
214
215
216
217
218
219
220
221
   
    // inputs and driver
    tailConfig.setDriver(CoGroupDriver.class);
    tailConfig.setDriverStrategy(DriverStrategy.CO_GROUP);
    tailConfig.addInputToGroup(0);
    tailConfig.addInputToGroup(1);
    tailConfig.setInputSerializer(recSerializer, 0);
    tailConfig.setInputSerializer(recSerializer, 1);
    tailConfig.setDriverComparator(fieldZeroComparator, 0);
    tailConfig.setDriverComparator(fieldZeroComparator, 1);
    tailConfig.setDriverPairComparator(pairComparatorFactory);
View Full Code Here
242
243
244
245
246
247
248
249
250
251
252
    // --------------- the output ---------------------

    JobOutputVertex output = JobGraphUtils.createFileOutput(jobGraph, "FinalOutput", degreeOfParallelism,
      numSubTasksPerInstance);
    TaskConfig outputConfig = new TaskConfig(output.getConfiguration());
    outputConfig.addInputToGroup(0);
    outputConfig.setInputSerializer(recSerializer, 0);
    outputConfig.setStubWrapper(new UserCodeClassWrapper<PageWithRankOutFormat>(PageWithRankOutFormat.class));
    outputConfig.setStubParameter(FileOutputFormat.FILE_PARAMETER_KEY, outputPath);
   
    // --------------- the auxiliaries ---------------------
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    {
      headConfig.setIterationId(ITERATION_ID);

      // input to iteration head
      headConfig.addInputToGroup(0);
      headConfig.setInputSerializer(serializer, 0);
      headConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
      headConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);

      // output into iteration
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.