Examples of DualInputPlanNode

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
      for (Channel c: sn.getBroadcastInputs()) {
        traverseChannel(c);
      }
    }
    else if (node instanceof DualInputPlanNode) {
      DualInputPlanNode dn = (DualInputPlanNode) node;
     
      if (!(dn.getOptimizerNode().getPactContract() instanceof DualInputOperator)) {
        throw new RuntimeException("Wrong operator type found in post pass.");
      }
     
      DualInputOperator<?, ?, ?, ?> dualInputOperator = (DualInputOperator<?, ?, ?, ?>) dn.getOptimizerNode().getPactContract();
     
      // parameterize the node's driver strategy
      if (dn.getDriverStrategy().getNumRequiredComparators() > 0) {
        dn.setComparator1(createComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dn.getKeysForInput1(),
          getSortOrders(dn.getKeysForInput1(), dn.getSortOrders())));
        dn.setComparator2(createComparator(dualInputOperator.getOperatorInfo().getSecondInputType(), dn.getKeysForInput2(),
            getSortOrders(dn.getKeysForInput2(), dn.getSortOrders())));

        dn.setPairComparator(createPairComparator(dualInputOperator.getOperatorInfo().getFirstInputType(),
            dualInputOperator.getOperatorInfo().getSecondInputType()));
       
      }
           
      traverseChannel(dn.getInput1());
      traverseChannel(dn.getInput2());
     
      // don't forget the broadcast inputs
      for (Channel c: dn.getBroadcastInputs()) {
        traverseChannel(c);
      }
     
    }
    // catch the sources of the iterative step functions
View Full Code Here

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
       
       
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
       
        assertTrue(mapper.getInput1().getTempMode().breaksPipeline());
      }
     
      {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setDegreeOfParallelism(64);
       
        DataSet<Long> initialSource = env.generateSequence(1, 10);
       
        Configuration conf= new Configuration();
        conf.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND);
        initialSource
          .map(new IdentityMapper<Long>())
          .cross(initialSource).withParameters(conf)
          .print();
       
       
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
       
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
       
        assertTrue(mapper.getInput2().getTempMode().breaksPipeline());
      }
     
      {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setDegreeOfParallelism(64);
       
        DataSet<Long> initialSource = env.generateSequence(1, 10);
       
        Configuration conf= new Configuration();
        conf.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST);
        initialSource
          .map(new IdentityMapper<Long>())
          .cross(initialSource).withParameters(conf)
          .print();
       
       
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
       
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
       
        assertTrue(mapper.getInput1().getTempMode().breaksPipeline());
      }
     
      {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setDegreeOfParallelism(64);
       
        DataSet<Long> initialSource = env.generateSequence(1, 10);
       
        Configuration conf= new Configuration();
        conf.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND);
        initialSource
          .map(new IdentityMapper<Long>())
          .cross(initialSource).withParameters(conf)
          .print();
       
       
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
       
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
       
        assertTrue(mapper.getInput2().getTempMode().breaksPipeline());
      }
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
View Full Code Here

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
      fail("The pact compiler is unable to compile this plan correctly.");
      return; // silence the compiler
    }
   
    OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
    DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
    DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
    SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
    SingleInputPlanNode deltaMapper = resolver.getNode(SOLUTION_DELTA_MAPPER_NAME);
   
    // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
    // the in-loop partitioning is before the final reducer
   
    // verify joinWithInvariant
    assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
    assertEquals(list0, joinWithInvariantNode.getKeysForInput1());
    assertEquals(list0, joinWithInvariantNode.getKeysForInput2());
   
    // verify joinWithSolutionSet
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
   
    // verify reducer
    assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
    assertEquals(list0, worksetReducer.getKeys(0));
   
View Full Code Here

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
      fail("The pact compiler is unable to compile this plan correctly.");
      return; // silence the compiler
    }
   
    OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
    DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
    DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
    SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
   
    // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
    // the in-loop partitioning is before the final reducer
   
    // verify joinWithInvariant
    assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
    assertEquals(list0, joinWithInvariantNode.getKeysForInput1());
    assertEquals(list0, joinWithInvariantNode.getKeysForInput2());
   
    // verify joinWithSolutionSet
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
   
    // verify reducer
    assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
    assertEquals(list0, worksetReducer.getKeys(0));
   
   
    // verify solution delta
    assertEquals(2, joinWithSolutionSetNode.getOutgoingChannels().size());
    assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
    assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(1).getShipStrategy());
   
    new NepheleJobGraphGenerator().compileJobGraph(oPlan);
  }
View Full Code Here

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
      fail("The pact compiler is unable to compile this plan correctly.");
      return; // silence the compiler
    }
   
    OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
    DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
    DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
    SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
   
    // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
    // the in-loop partitioning is before the final reducer
   
    // verify joinWithInvariant
    assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
    assertEquals(list0, joinWithInvariantNode.getKeysForInput1());
    assertEquals(list0, joinWithInvariantNode.getKeysForInput2());
   
    // verify joinWithSolutionSet
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
   
    // verify reducer
    assertEquals(ShipStrategyType.FORWARD, worksetReducer.getInput().getShipStrategy());
    assertEquals(list0, worksetReducer.getKeys(0));
   
   
    // verify solution delta
    assertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
   
    new NepheleJobGraphGenerator().compileJobGraph(oPlan);
  }
View Full Code Here

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

64
65
66
67
68
69
70
71
72
73
74
75
76
   
    try {
      OptimizedPlan oPlan = compileNoStats(plan);
      OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan);
     
      DualInputPlanNode crossPlanNode = resolver.getNode("Cross");
      Channel in1 = crossPlanNode.getInput1();
      Channel in2 = crossPlanNode.getInput2();
     
      assertEquals(ShipStrategyType.FORWARD, in1.getShipStrategy());
      assertEquals(ShipStrategyType.BROADCAST, in2.getShipStrategy());
    } catch(CompilerException ce) {
      ce.printStackTrace();
View Full Code Here

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

96
97
98
99
100
101
102
103
104
105
106
107
108
   
    try {
      OptimizedPlan oPlan = compileNoStats(plan);
      OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan);
     
      DualInputPlanNode crossPlanNode = resolver.getNode("Cross");
      Channel in1 = crossPlanNode.getInput1();
      Channel in2 = crossPlanNode.getInput2();
     
      assertEquals(ShipStrategyType.BROADCAST, in1.getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, in2.getShipStrategy());
    } catch(CompilerException ce) {
      ce.printStackTrace();
View Full Code Here

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

907
908
909
910
911
912
913
914
915
916
917
918
919
      toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
      toJoin1.setLocalStrategy(LocalStrategy.NONE);
      toJoin2.setShipStrategy(ShipStrategyType.FORWARD);
      toJoin2.setLocalStrategy(LocalStrategy.NONE);
     
      DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
     
      FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties());
      assertEquals(NO_PARTIAL_SOLUTION, report);
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
View Full Code Here

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
      SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
     
      Channel toJoin1 = new Channel(map1);
      Channel toJoin2 = new Channel(map2);
     
      DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
     
      Channel toAfterJoin = new Channel(join);
      toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD);
      toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
      SingleInputPlanNode afterJoin = new SingleInputPlanNode(getMapNode(), "After Join Mapper", toAfterJoin, DriverStrategy.MAP);
     
      // attach some properties to the non-relevant input
      {
        toMap2.setShipStrategy(ShipStrategyType.BROADCAST);
        toMap2.setLocalStrategy(LocalStrategy.SORT, new FieldList(2, 7), new boolean[] {true, true});
       
        RequestedGlobalProperties joinGp = new RequestedGlobalProperties();
        joinGp.setFullyReplicated();
       
        RequestedLocalProperties joinLp = new RequestedLocalProperties();
        joinLp.setOrdering(new Ordering(2, null, Order.ASCENDING).appendOrdering(7, null, Order.ASCENDING));
       
        toJoin2.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin2.setLocalStrategy(LocalStrategy.NONE);
        toJoin2.setRequiredGlobalProps(joinGp);
        toJoin2.setRequiredLocalProps(joinLp);
      }
     
      // ------------------------------------------------------------------------------------
     
      // no properties from the partial solution, no required properties
      {
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
       
        GlobalProperties gp = new GlobalProperties();
        LocalProperties lp = LocalProperties.EMPTY;
       
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
     
      // some properties from the partial solution, no required properties
      {
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
       
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
       
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
     
     
      // produced properties match relevant input
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
       
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
       
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(2));
       
        toJoin1.setRequiredGlobalProps(rgp);
        toJoin1.setRequiredLocalProps(rlp);
       
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
       
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
     
      // produced properties do not match relevant input
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
       
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
       
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(1, 2, 3));
       
        toJoin1.setRequiredGlobalProps(rgp);
        toJoin1.setRequiredLocalProps(rlp);
       
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
       
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(NOT_MET, report);
      }
     
      // produced properties overridden before join
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
       
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
       
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(2, 1));
       
        toMap1.setRequiredGlobalProps(rgp);
        toMap1.setRequiredLocalProps(rlp);
       
        toJoin1.setRequiredGlobalProps(null);
        toJoin1.setRequiredLocalProps(null);
       
        toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 1));
        toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] {true, false});
       
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(MET, report);
      }
     
      // produced properties before join match, after join match as well
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
       
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
       
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(2, 1));
       
        toMap1.setRequiredGlobalProps(null);
        toMap1.setRequiredLocalProps(null);
       
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
       
        toJoin1.setRequiredGlobalProps(rgp);
        toJoin1.setRequiredLocalProps(rlp);
     
        toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD);
        toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
       
        toAfterJoin.setRequiredGlobalProps(rgp);
        toAfterJoin.setRequiredLocalProps(rlp);
       
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
     
      // produced properties before join match, after join do not match
      {
View Full Code Here

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
     
      Channel toJoin2 = new Channel(map2);
      toJoin2.setShipStrategy(ShipStrategyType.FORWARD);
      toJoin2.setLocalStrategy(LocalStrategy.NONE);
     
      DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
     
      Channel toAfterJoin = new Channel(join);
      toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD);
      toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
      SingleInputPlanNode afterJoin = new SingleInputPlanNode(getMapNode(), "After Join Mapper", toAfterJoin, DriverStrategy.MAP);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.