Package org.apache.flink.api.common.aggregators

Examples of org.apache.flink.api.common.aggregators.LongSumAggregator

119
120
121
122
123
124
125
126
127
128
129
      IterativeDataSet<Tuple2<Long, Long>> iteration =
          initialSolutionSet.iterate(MAX_ITERATIONS);

      // register the convergence criterion
      iteration.registerAggregationConvergenceCriterion(UPDATED_ELEMENTS,
          new LongSumAggregator(), new UpdatedElementsConvergenceCriterion(convergence_threshold));

      DataSet<Tuple2<Long, Long>> verticesWithNewComponents = iteration.join(edges).where(0).equalTo(0)
          .with(new NeighborWithComponentIDJoin())
          .groupBy(0).reduceGroup(new MinimumReduce());
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
        DataSet<Integer> initialSolutionSet = CollectionDataSets.getIntegerDataSet(env);
        IterativeDataSet<Integer> iteration = initialSolutionSet.iterate(MAX_ITERATIONS);

        // register aggregator
        LongSumAggregator aggr = new LongSumAggregator();
        iteration.registerAggregator(NEGATIVE_ELEMENTS_AGGR, aggr);
       
        // register convergence criterion
        iteration.registerAggregationConvergenceCriterion(NEGATIVE_ELEMENTS_AGGR, aggr,
            new NegativeElementsConvergenceCriterion());
       
        DataSet<Integer> updatedDs = iteration.map(new SubtractOneMap());
        iteration.closeWith(updatedDs).writeAsText(resultPath);
        env.execute();

        // return expected result
        return "-3\n" + "-2\n" + "-2\n" + "-1\n" + "-1\n"
             + "-1\n" + "0\n" + "0\n" + "0\n" + "0\n"
             + "1\n" + "1\n" + "1\n" + "1\n" + "1\n";
      }
      case 2: {
        /*
         * Test aggregator with parameter for iterate
         */
       
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setDegreeOfParallelism(DOP);

        DataSet<Integer> initialSolutionSet = CollectionDataSets.getIntegerDataSet(env);
        IterativeDataSet<Integer> iteration = initialSolutionSet.iterate(MAX_ITERATIONS);

        // register aggregator
        LongSumAggregatorWithParameter aggr = new LongSumAggregatorWithParameter(0);
        iteration.registerAggregator(NEGATIVE_ELEMENTS_AGGR, aggr);
       
        // register convergence criterion
        iteration.registerAggregationConvergenceCriterion(NEGATIVE_ELEMENTS_AGGR, aggr,
            new NegativeElementsConvergenceCriterion());
       
        DataSet<Integer> updatedDs = iteration.map(new SubtractOneMapWithParam());
        iteration.closeWith(updatedDs).writeAsText(resultPath);
        env.execute();
       
        // return expected result
        return "-3\n" + "-2\n" + "-2\n" + "-1\n" + "-1\n"
             + "-1\n" + "0\n" + "0\n" + "0\n" + "0\n"
             + "1\n" + "1\n" + "1\n" + "1\n" + "1\n";
      }
      case 3: {
        /*
         * Test convergence criterion with parameter for iterate
         */
       
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setDegreeOfParallelism(DOP);

        DataSet<Integer> initialSolutionSet = CollectionDataSets.getIntegerDataSet(env);
        IterativeDataSet<Integer> iteration = initialSolutionSet.iterate(MAX_ITERATIONS);

        // register aggregator
        LongSumAggregator aggr = new LongSumAggregator();
        iteration.registerAggregator(NEGATIVE_ELEMENTS_AGGR, aggr);
       
        // register convergence criterion
        iteration.registerAggregationConvergenceCriterion(NEGATIVE_ELEMENTS_AGGR, aggr,
            new NegativeElementsConvergenceCriterionWithParam(3));
       
        DataSet<Integer> updatedDs = iteration.map(new SubtractOneMap());
        iteration.closeWith(updatedDs).writeAsText(resultPath);
        env.execute();
       
        // return expected result
        return "-3\n" + "-2\n" + "-2\n" + "-1\n" + "-1\n"
             + "-1\n" + "0\n" + "0\n" + "0\n" + "0\n"
             + "1\n" + "1\n" + "1\n" + "1\n" + "1\n";
      }
      case 4: {
        /*
         * Test aggregator without parameter for iterateDelta
         */
       
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setDegreeOfParallelism(DOP);
       
        DataSet<Tuple2<Integer, Integer>> initialSolutionSet = CollectionDataSets.getIntegerDataSet(env).map(new TupleMakerMap());
           
        DeltaIteration<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> iteration = initialSolutionSet.iterateDelta(
            initialSolutionSet, MAX_ITERATIONS, 0);

        // register aggregator
        LongSumAggregator aggr = new LongSumAggregator();
        iteration.registerAggregator(NEGATIVE_ELEMENTS_AGGR, aggr);
       
        DataSet<Tuple2<Integer, Integer>> updatedDs = iteration.getWorkset().map(new AggregateMapDelta());
       
        DataSet<Tuple2<Integer, Integer>> newElements = updatedDs.join(iteration.getSolutionSet())
            .where(0).equalTo(0).flatMap(new UpdateFilter());
       
        DataSet<Tuple2<Integer, Integer>> iterationRes = iteration.closeWith(newElements, newElements);
        DataSet<Integer> result = iterationRes.map(new ProjectSecondMapper());
        result.writeAsText(resultPath);
       
        env.execute();
       
        // return expected result
        return "1\n" + "2\n" + "2\n" + "3\n" + "3\n"
             + "3\n" + "4\n" + "4\n" + "4\n" + "4\n"
             + "5\n" + "5\n" + "5\n" + "5\n" + "5\n";
       
      }
      case 5: {
        /*
         * Test aggregator with parameter for iterateDelta
         */
       
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setDegreeOfParallelism(DOP);
       
        DataSet<Tuple2<Integer, Integer>> initialSolutionSet = CollectionDataSets.getIntegerDataSet(env).map(new TupleMakerMap());
           
        DeltaIteration<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> iteration = initialSolutionSet.iterateDelta(
            initialSolutionSet, MAX_ITERATIONS, 0);

        // register aggregator
        LongSumAggregator aggr = new LongSumAggregatorWithParameter(4);
        iteration.registerAggregator(NEGATIVE_ELEMENTS_AGGR, aggr);
       
        DataSet<Tuple2<Integer, Integer>> updatedDs = iteration.getWorkset().map(new AggregateMapDelta());
       
        DataSet<Tuple2<Integer, Integer>> newElements = updatedDs.join(iteration.getSolutionSet())
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
        vertexIteration.addBroadcastSetForUpdateFunction(BC_SET_UPDATES_NAME, bcUpdate);
       
        vertexIteration.setName(ITERATION_NAME);
        vertexIteration.setParallelism(ITERATION_DOP);
       
        vertexIteration.registerAggregator(AGGREGATOR_NAME, new LongSumAggregator());
       
        result = initialVertices.runOperation(vertexIteration);
      }
     
     
View Full Code Here
154
155
156
157
158
159
160
161
162
163
164
        vertexIteration.addBroadcastSetForUpdateFunction(BC_SET_UPDATES_NAME, bcVar);
       
        vertexIteration.setName(ITERATION_NAME);
        vertexIteration.setParallelism(ITERATION_DOP);
       
        vertexIteration.registerAggregator(AGGREGATOR_NAME, new LongSumAggregator());
       
        result = initialVertices.runOperation(vertexIteration);
      }
     
     
View Full Code Here
285
286
287
288
289
290
291
292
293
294
      headConfig.setDriverComparator(comparator, 1);
      headConfig.setDriverPairComparator(pairComparator);
      headConfig.setRelativeMemoryDriver(MEM_FRAC_PER_CONSUMER);

      headConfig.addIterationAggregator(
        WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new LongSumAggregator());
    }

    return head;
  }
View Full Code Here
353
354
355
356
357
358
359
360
361
362
363
    AbstractJobVertex sync = JobGraphUtils.createSync(jobGraph, numSubTasks);
    TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setNumberOfIterations(maxIterations);
    syncConfig.setIterationId(ITERATION_ID);
    syncConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME,
      new LongSumAggregator());
    syncConfig.setConvergenceCriterion(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME,
      new WorksetEmptyConvergenceCriterion());

    return sync;
  }
View Full Code Here
1395
1396
1397
1398
1399
1400
1401
1402
1403
   
    if (convCriterion != null || convAggName != null) {
      throw new CompilerException("Error: Cannot use custom convergence criterion with workset iteration. Workset iterations have implicit convergence criterion where workset is empty.");
    }
   
    headConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new LongSumAggregator());
    syncConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new LongSumAggregator());
    syncConfig.setConvergenceCriterion(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new WorksetEmptyConvergenceCriterion());
  }
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
        DataSet<Tuple2<Double, String>> initialWorkSet = env.fromElements(new Tuple2<Double, String>(1.23, "abc"));
       
        DeltaIteration<Tuple3<Double, Long, String>, Tuple2<Double, String>> iteration = initialSolutionSet.iterateDelta(initialWorkSet, NUM_ITERATIONS, ITERATION_KEYS);
        iteration.name(ITERATION_NAME).parallelism(ITERATION_DOP);
       
        iteration.registerAggregator(AGGREGATOR_NAME, new LongSumAggregator());
       
        // test that multiple workset consumers are supported
        DataSet<Tuple2<Double, String>> worksetSelfJoin =
          iteration.getWorkset()
            .map(new IdentityMapper<Tuple2<Double,String>>())
View Full Code Here
TOP

Related Classes of org.apache.flink.api.common.aggregators.LongSumAggregator

  • org.apache.flink.api.java.operators.translation.DeltaIterationTranslationTest
  • org.apache.flink.compiler.plantranslate.NepheleJobGraphGenerator
  • org.apache.flink.spargel.java.SpargelTranslationTest
  • org.apache.flink.test.iterative.aggregators.AggregatorsITCase$AggregatorProgs
  • org.apache.flink.test.iterative.aggregators.ConnectedComponentsWithParametrizableConvergenceITCase$ConnectedComponentsWithConvergenceProgram
  • org.apache.flink.test.iterative.nephele.ConnectedComponentsNepheleITCase
  • org.apache.flink.types.LongValue
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.