Package eu.stratosphere.api.common.operators.util

Examples of eu.stratosphere.api.common.operators.util.FieldList

219
220
221
222
223
224
225
226
227
228
229
230
231
      // check that both reduce and combiner have the same strategy
      assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
      assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combineNode.getDriverStrategy());
     
      // check the keys
      assertEquals(new FieldList(1), reduceNode.getKeys());
      assertEquals(new FieldList(1), combineNode.getKeys());
      assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
     
      // check DOP
      assertEquals(6, sourceNode.getDegreeOfParallelism());
      assertEquals(6, combineNode.getDegreeOfParallelism());
      assertEquals(8, reduceNode.getDegreeOfParallelism());
View Full Code Here
276
277
278
279
280
281
282
283
284
285
286
287
     
      // check that both reduce and combiner have the same strategy
      assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
     
      // check the keys
      assertEquals(new FieldList(0), reduceNode.getKeys());
      assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
     
      // check DOP
      assertEquals(6, sourceNode.getDegreeOfParallelism());
      assertEquals(6, keyExtractor.getDegreeOfParallelism());
     
View Full Code Here
340
341
342
343
344
345
346
347
348
349
350
351
352
      // check that both reduce and combiner have the same strategy
      assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
      assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combineNode.getDriverStrategy());
     
      // check the keys
      assertEquals(new FieldList(0), reduceNode.getKeys());
      assertEquals(new FieldList(0), combineNode.getKeys());
      assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
     
      // check DOP
      assertEquals(6, sourceNode.getDegreeOfParallelism());
      assertEquals(6, keyExtractor.getDegreeOfParallelism());
      assertEquals(6, combineNode.getDegreeOfParallelism());
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
      assertTrue(ssDelta instanceof DualInputPlanNode); // this is only true if the update functions preserves the partitioning
     
      DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
      assertEquals(DEFAULT_PARALLELISM, ssJoin.getDegreeOfParallelism());
      assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
      assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());
     
      // check the workset set join
      DualInputPlanNode edgeJoin = (DualInputPlanNode) ssJoin.getInput1().getSource();
      assertEquals(DEFAULT_PARALLELISM, edgeJoin.getDegreeOfParallelism());
      assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput2().getShipStrategy());
      assertTrue(edgeJoin.getInput1().getTempMode().isCached());
     
      assertEquals(new FieldList(0), edgeJoin.getInput1().getShipStrategyKeys());
     
      // check that the initial partitioning is pushed out of the loop
      assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput2().getShipStrategy());
      assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
      assertEquals(new FieldList(0), iteration.getInput2().getShipStrategyKeys());
     
      // check that the initial workset sort is outside the loop
      assertEquals(LocalStrategy.SORT, iteration.getInput2().getLocalStrategy());
      assertEquals(new FieldList(0), iteration.getInput2().getLocalStrategyKeys());
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail(e.getMessage());
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
      assertTrue(ssDelta instanceof DualInputPlanNode); // this is only true if the update functions preserves the partitioning
     
      DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
      assertEquals(DEFAULT_PARALLELISM, ssJoin.getDegreeOfParallelism());
      assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
      assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());
     
      // check the workset set join
      DualInputPlanNode edgeJoin = (DualInputPlanNode) ssJoin.getInput1().getSource();
      assertEquals(DEFAULT_PARALLELISM, edgeJoin.getDegreeOfParallelism());
      assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput2().getShipStrategy());
      assertTrue(edgeJoin.getInput1().getTempMode().isCached());
     
      assertEquals(new FieldList(0), edgeJoin.getInput1().getShipStrategyKeys());
     
      // check that the initial partitioning is pushed out of the loop
      assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput2().getShipStrategy());
      assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
      assertEquals(new FieldList(0), iteration.getInput2().getShipStrategyKeys());
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail(e.getMessage());
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, reducer.getInput().getShipStrategy());
   
    Channel c = reducer.getInput();
    Assert.assertEquals(LocalStrategy.SORT, c.getLocalStrategy());
   
    FieldList ship = new FieldList(2);
    FieldList local = new FieldList(2);
    local.add(5);
    Assert.assertEquals(ship, c.getShipStrategyKeys());
    Assert.assertEquals(local, c.getLocalStrategyKeys());
    Assert.assertTrue(c.getLocalStrategySortOrder()[0] == reducer.getSortOrders()[0]);
   
    // check that we indeed sort descending
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
    Channel c2 = coGroupNode.getInput2();
   
    Assert.assertEquals(LocalStrategy.SORT, c1.getLocalStrategy());
    Assert.assertEquals(LocalStrategy.SORT, c2.getLocalStrategy());
   
    FieldList ship1 = new FieldList(new int[] {3, 0});
    FieldList ship2 = new FieldList(new int[] {6, 0});
   
    FieldList local1 = new FieldList(new int[] {3, 0, 5});
    FieldList local2 = new FieldList(new int[] {6, 0, 1, 4});
   
    Assert.assertEquals(ship1, c1.getShipStrategyKeys());
    Assert.assertEquals(ship2, c2.getShipStrategyKeys());
    Assert.assertEquals(local1, c1.getLocalStrategyKeys());
    Assert.assertEquals(local2, c2.getLocalStrategyKeys());
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
      // the in-loop partitioning is before the final reducer
     
      // verify joinWithInvariant
      assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
     
      // verify joinWithSolutionSet
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(0, 1), joinWithSolutionSetNode.getKeysForInput1());
     
     
      // verify reducer
      assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
      assertEquals(new FieldList(1, 2), worksetReducer.getKeys());
     
      // currently, the system may partition before or after the mapper
      ShipStrategyType ss1 = deltaMapper.getInput().getShipStrategy();
      ShipStrategyType ss2 = deltaMapper.getOutgoingChannels().get(0).getShipStrategy();
     
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
      // the in-loop partitioning is before the final reducer
     
      // verify joinWithInvariant
      assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
     
      // verify joinWithSolutionSet
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(0, 1), joinWithSolutionSetNode.getKeysForInput1());
     
      // verify reducer
      assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
      assertEquals(new FieldList(1, 2), worksetReducer.getKeys());
     
      // verify solution delta
      assertEquals(2, joinWithSolutionSetNode.getOutgoingChannels().size());
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(1).getShipStrategy());
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
      // the in-loop partitioning is before the final reducer
     
      // verify joinWithInvariant
      assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
     
      // verify joinWithSolutionSet
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(0, 1), joinWithSolutionSetNode.getKeysForInput1());
     
      // verify reducer
      assertEquals(ShipStrategyType.FORWARD, worksetReducer.getInput().getShipStrategy());
      assertEquals(new FieldList(1, 2), worksetReducer.getKeys());
     
     
      // verify solution delta
      assertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
      assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
View Full Code Here
TOP

Related Classes of eu.stratosphere.api.common.operators.util.FieldList

  • eu.stratosphere.compiler.dag.TwoInputNode
  • eu.stratosphere.compiler.dag.WorksetIterationNode
  • eu.stratosphere.compiler.dataproperties.LocalProperties
  • eu.stratosphere.compiler.dataproperties.RequestedLocalProperties
  • eu.stratosphere.compiler.util.Utils
  • eu.stratosphere.pact.compiler.GroupOrderTest
  • eu.stratosphere.pact.compiler.GroupReduceCompilationTest
  • eu.stratosphere.pact.compiler.ReduceCompilationTest
  • eu.stratosphere.pact.compiler.WorksetIterationsJavaApiCompilerTest
  • eu.stratosphere.spargel.java.SpargelCompilerTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.