Package eu.stratosphere.compiler.plan

Examples of eu.stratosphere.compiler.plan.OptimizedPlan

531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
      sinks.add(sinkB);
     
      // return the PACT plan
      Plan plan = new Plan(sinks, "Plans With Multiple Data Sinks");
     
      OptimizedPlan oPlan = compileNoStats(plan);
     
      // ---------- check the optimizer plan ----------
     
      // number of sinks
      Assert.assertEquals("Wrong number of data sinks.", 2, oPlan.getDataSinks().size());
     
      // sinks contain all sink paths
      Set<String> allSinks = new HashSet<String>();
      allSinks.add(out1Path);
      allSinks.add(out2Path);
     
      for (SinkPlanNode n : oPlan.getDataSinks()) {
        String path = ((FileDataSink) n.getSinkNode().getPactContract()).getFilePath();
        Assert.assertTrue("Invalid data sink.", allSinks.remove(path));
      }
     
      // ---------- compile plan to nephele job graph to verify that no error is thrown ----------
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
    Plan plan = new Plan(output);
    plan.setDefaultParallelism(4);

    // optimize and compile plan
    PactCompiler pc = new PactCompiler(new DataStatistics());
    OptimizedPlan op = pc.compile(plan);
   
    // return job graph of failing job
    NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
    return jgg.compileJobGraph(op);
  }
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
    Plan plan = new Plan(output);
    plan.setDefaultParallelism(4);

    // optimize and compile plan
    PactCompiler pc = new PactCompiler(new DataStatistics());
    OptimizedPlan op = pc.compile(plan);

    // return job graph of working job
    NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
    return jgg.compileJobGraph(op);
  }
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
    sink.setInput(reduce2);
   
    Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
   
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);
   
    // check the optimized Plan
    // when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
    // because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
    // mapper respectively reducer
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
    SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
   
    ShipStrategyType mapIn = map2Node.getInput().getShipStrategy();
    ShipStrategyType redIn = red2Node.getInput().getShipStrategy();
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
    sink.setInput(reduce2);
   
    Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
   
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);
   
    // check the optimized Plan
    // when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
    // because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
    // mapper respectively reducer
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
    SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
   
    ShipStrategyType mapIn = map2Node.getInput().getShipStrategy();
    ShipStrategyType reduceIn = red2Node.getInput().getShipStrategy();
View Full Code Here
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
    sink.setInput(reduce2);
   
    Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
   
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);
   
    // check the optimized Plan
    // when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
    // because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
    // mapper respectively reducer
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
    SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
   
    ShipStrategyType mapIn = map2Node.getInput().getShipStrategy();
    ShipStrategyType reduceIn = red2Node.getInput().getShipStrategy();
View Full Code Here
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
    sink.setInput(reduce2);
   
    Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
   
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);
   
    // check the optimized Plan
    // when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
    // because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
    // mapper respectively reducer
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
   
    Assert.assertEquals("The Reduce 2 Node has an invalid local strategy.", LocalStrategy.SORT, red2Node.getInput().getLocalStrategy());
  }
View Full Code Here
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
   
   
    // return the PACT plan
    Plan plan = new Plan(sink, "Partition on DoP Change");
   
    OptimizedPlan oPlan = compileNoStats(plan);
   
    NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
   
    //Compile plan to verify that no error is thrown
    jobGen.compileJobGraph(oPlan);
   
    oPlan.accept(new Visitor<PlanNode>() {
     
      @Override
      public boolean preVisit(PlanNode visitable) {
        if (visitable instanceof DualInputPlanNode) {
          DualInputPlanNode node = (DualInputPlanNode) visitable;
View Full Code Here
68
69
70
71
72
73
74
75
76
    sink.setInput(reduce3);
   
    Plan plan = new Plan(sink, "Test Temp Task");
    plan.setDefaultParallelism(DEFAULT_PARALLELISM);
   
    OptimizedPlan oPlan = compileNoStats(plan);
    NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
    jobGen.compileJobGraph(oPlan);
  }
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
    Plan plan = new Plan(sink, "AllReduce Test");
    plan.setDefaultParallelism(DEFAULT_PARALLELISM);
   
   
    try {
      OptimizedPlan oPlan = compileNoStats(plan);
      NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
      jobGen.compileJobGraph(oPlan);
    } catch(CompilerException ce) {
      ce.printStackTrace();
      fail("The pact compiler is unable to compile this plan correctly");
View Full Code Here
TOP

Related Classes of eu.stratosphere.compiler.plan.OptimizedPlan

  • eu.stratosphere.client.LocalExecutor
  • eu.stratosphere.client.program.ContextEnvironment
  • eu.stratosphere.client.RemoteExecutor
  • eu.stratosphere.client.web.JobSubmissionServlet
  • eu.stratosphere.compiler.PactCompiler
  • eu.stratosphere.compiler.PactCompiler$PlanFinalizer
  • eu.stratosphere.pact.compiler.AdditionalOperatorsTest
  • eu.stratosphere.pact.compiler.BranchingPlansCompilerTest
  • eu.stratosphere.pact.compiler.DOPChangeTest
  • eu.stratosphere.pact.compiler.GroupOrderTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.