Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.FileDataSource

253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
  @SuppressWarnings("unchecked")
  private static Plan getPlanForWorksetConnectedComponentsWithSolutionSetAsFirstInput(
      int numSubTasks, String verticesInput, String edgeInput, String output, int maxIterations)
  {
    // create DataSourceContract for the vertices
    FileDataSource initialVertices = new FileDataSource(new CsvInputFormat(' ', LongValue.class), verticesInput, "Vertices");
   
    MapOperator verticesWithId = MapOperator.builder(DuplicateLongMap.class).input(initialVertices).name("Assign Vertex Ids").build();
   
    DeltaIteration iteration = new DeltaIteration(0, "Connected Components Iteration");
    iteration.setInitialSolutionSet(verticesWithId);
    iteration.setInitialWorkset(verticesWithId);
    iteration.setMaximumNumberOfIterations(maxIterations);
   
    // create DataSourceContract for the edges
    FileDataSource edges = new FileDataSource(new CsvInputFormat(' ', LongValue.class, LongValue.class), edgeInput, "Edges");

    // create CrossOperator for distance computation
    JoinOperator joinWithNeighbors = JoinOperator.builder(new NeighborWithComponentIDJoin(), LongValue.class, 0, 0)
        .input1(iteration.getWorkset())
        .input2(edges)
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
61
  @SuppressWarnings({ "deprecation", "unchecked" })
  @Test
  public void testUnionPropertyOldApiPropagation() {
    // construct the plan

    FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), IN_FILE);
    FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), IN_FILE);
   
    ReduceOperator redA = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
      .input(sourceA)
      .build();
    ReduceOperator redB = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
 
      // construct the plan
      final String out1Path = "file:///test/1";
      final String out2Path = "file:///test/2";
 
      FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);
 
      MapOperator mapA = MapOperator.builder(IdentityMap.class).input(sourceA).name("Map A").build();
      MapOperator mapC = MapOperator.builder(IdentityMap.class).input(mapA).name("Map C").build();
 
      FileDataSink[] sinkA = new FileDataSink[SINKS];
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
      // construct the plan
      final String out1Path = "file:///test/1";
      final String out2Path = "file:///test/2";
      final String out3Path = "file:///test/3";
 
      FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);

      MapOperator mapA = MapOperator.builder(IdentityMap.class).input(sourceA).name("Map A").build();
      MapOperator mapB = MapOperator.builder(IdentityMap.class).input(mapA).name("Map B").build();
      MapOperator mapC = MapOperator.builder(IdentityMap.class).input(mapA).name("Map C").build();
     
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
   */
  @Test
  public void testBranchingSourceMultipleTimes() {
    try {
      // construct the plan
      FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), IN_FILE);
     
      JoinOperator mat1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(sourceA)
        .input2(sourceA)
        .build();
View Full Code Here
288
289
290
291
292
293
294
295
296
297
298
299
300
      // construct the plan
      final String out1Path = "file:///test/1";
      final String out2Path = "file:///test/2";
      final String out3Path = "file:///test/3";
 
      FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), IN_FILE);
      FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), IN_FILE);
      FileDataSource sourceC = new FileDataSource(new DummyInputFormat(), IN_FILE);
     
      CoGroupOperator co = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
        .input1(sourceA)
        .input2(sourceB)
        .build();
View Full Code Here
357
358
359
360
361
362
363
364
365
366
367
368
369
 
  @Test
  public void testBranchEachContractType() {
    try {
      // construct the plan
      FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), "file:///test/file1", "Source A");
      FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), "file:///test/file2", "Source B");
      FileDataSource sourceC = new FileDataSource(new DummyInputFormat(), "file:///test/file3", "Source C");
     
      MapOperator map1 = MapOperator.builder(new IdentityMap()).input(sourceA).name("Map 1").build();
     
      ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(map1)
View Full Code Here
453
454
455
456
457
458
459
460
461
462
463
464
  @Test
  public void testBranchingUnion() {
    try {
      // construct the plan
      FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE);
      FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE);
     
      JoinOperator mat1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(source1)
        .input2(source2)
        .name("Match 1")
View Full Code Here
519
520
521
522
523
524
525
526
527
528
529
    try {
      // construct the plan
      final String out1Path = "file:///test/1";
      final String out2Path = "file:///test/2";
 
      FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);
     
      FileDataSink sinkA = new FileDataSink(DummyOutputFormat.class, out1Path, sourceA);
      FileDataSink sinkB = new FileDataSink(DummyOutputFormat.class, out2Path, sourceA);
     
      List<FileDataSink> sinks = new ArrayList<FileDataSink>();
View Full Code Here
572
573
574
575
576
577
578
579
580
581
582
583
  public void testSimpleDisjointPlan() {
    // construct the plan
    final String out1Path = "file:///test/1";
    final String out2Path = "file:///test/2";

    FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);
    FileDataSource sourceB = new FileDataSource(DummyInputFormat.class, IN_FILE);
   
    FileDataSink sinkA = new FileDataSink(DummyOutputFormat.class, out1Path, sourceA);
    FileDataSink sinkB = new FileDataSink(DummyOutputFormat.class, out2Path, sourceB);
   
    List<FileDataSink> sinks = new ArrayList<FileDataSink>();
View Full Code Here
TOP

Related Classes of eu.stratosphere.api.java.record.operators.FileDataSource

  • eu.stratosphere.pact.compiler.AdditionalOperatorsTest
  • eu.stratosphere.pact.compiler.BranchingPlansCompilerTest
  • eu.stratosphere.pact.compiler.DOPChangeTest
  • eu.stratosphere.pact.compiler.GroupOrderTest
  • eu.stratosphere.pact.compiler.HardPlansCompilationTest
  • eu.stratosphere.pact.compiler.ReduceAllTest
  • eu.stratosphere.pact.compiler.UnionPropertyPropagationTest
  • eu.stratosphere.pact.compiler.WorksetIterationsRecordApiCompilerTest
  • eu.stratosphere.test.accumulators.AccumulatorITCase
  • eu.stratosphere.test.accumulators.AccumulatorIterativeITCase
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.