Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.FileDataSource

612
613
614
615
616
617
618
619
620
621
622
623
    final String out1Path = "file:///test/1";
    final String out2Path = "file:///test/2";
    final String out3Path = "file:///test/3";
    final String out4Path = "file:///test/4";

    FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);
    FileDataSource sourceB = new FileDataSource(DummyInputFormat.class, IN_FILE);
   
    FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, out1Path, sourceA, "1");
    FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, out2Path, sourceB, "2");
    FileDataSink sink3 = new FileDataSink(DummyOutputFormat.class, out3Path, sourceA, "3");
    FileDataSink sink4 = new FileDataSink(DummyOutputFormat.class, out4Path, sourceB, "4");
View Full Code Here
641
642
643
644
645
646
647
648
649
650
651
    }
  }
 
  @Test
  public void testBranchAfterIteration() {
    FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 2");
   
    BulkIteration iteration = new BulkIteration("Loop");
    iteration.setInput(sourceA);
    iteration.setMaximumNumberOfIterations(10);
   
View Full Code Here
674
675
676
677
678
679
680
681
682
683
684
685
    }
  }
 
  @Test
  public void testBranchBeforeIteration() {
    FileDataSource source1 = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 1");
    FileDataSource source2 = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 2");
   
    BulkIteration iteration = new BulkIteration("Loop");
    iteration.setInput(source2);
    iteration.setMaximumNumberOfIterations(10);
   
View Full Code Here
722
723
724
725
726
727
728
729
730
731
732
733
   *         (SINK 3)     (CROSS => NEXT PARTIAL SOLUTION)
   * </pre>
   */
  @Test
  public void testClosure() {
    FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 1");
    FileDataSource sourceB = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 2");

    FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceA, "Sink 1");
    FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceB, "Sink 2");

    BulkIteration iteration = new BulkIteration("Loop");
View Full Code Here
769
770
771
772
773
774
775
776
777
778
779
780
781
   *                (JOIN => SOLUTION SET DELTA)
   * </pre>
   */
  @Test
  public void testClosureDeltaIteration() {
    FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 1");
    FileDataSource sourceB = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 2");
    FileDataSource sourceC = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 3");

    FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceA, "Sink 1");
    FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceC, "Sink 2");

    DeltaIteration iteration = new DeltaIteration(0, "Loop");
View Full Code Here
831
832
833
834
835
836
837
838
839
840
841
   * <p/>
   * </prev>
   */
  @Test
  public void testDeltaIterationWithStaticInput() {
    FileDataSource source = new FileDataSource(DummyInputFormat.class, IN_FILE, "source");

    MapOperator mappedSource = MapOperator.builder(IdentityMap.class).
        input(source).
        name("Identity mapped source").
        build();
View Full Code Here
894
895
896
897
898
899
900
901
902
903
904
   * <p/>
   * </prev>
   */
  @Test
  public void testIterationWithStaticInput() {
    FileDataSource source = new FileDataSource(DummyInputFormat.class, IN_FILE, "source");

    MapOperator mappedSource = MapOperator.builder(IdentityMap.class).
        input(source).
        name("Identity mapped source").
        build();
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
204
    final String edgeInput = args.length > 1 ? args[1] : "";
    final String output    = args.length > 2 ? args[2] : "";
    final char delimiter   = args.length > 3 ? (char) Integer.parseInt(args[3]) : ',';
   

    FileDataSource edges = new FileDataSource(new EdgeInputFormat(), edgeInput, "Input Edges");
    edges.setParameter(EdgeInputFormat.ID_DELIMITER_CHAR, delimiter);
   
    MapOperator projectEdge = MapOperator.builder(new ProjectEdge())
      .input(edges).name("Project Edge").build();
   
    ReduceOperator edgeCounter = ReduceOperator.builder(new CountEdges(), IntValue.class, 0)
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
    final String edgeInput = (args.length > 2 ? args[2] : "");
    final String output = (args.length > 3 ? args[3] : "");
    final int maxIterations = (args.length > 4 ? Integer.parseInt(args[4]) : 1);

    // data source for initial vertices
    FileDataSource initialVertices = new FileDataSource(new CsvInputFormat(' ', LongValue.class), verticesInput, "Vertices");
   
    MapOperator verticesWithId = MapOperator.builder(DuplicateLongMap.class).input(initialVertices).name("Assign Vertex Ids").build();
   
    DeltaIteration iteration = new DeltaIteration(0, "Connected Components Iteration");
    iteration.setInitialSolutionSet(verticesWithId);
    iteration.setInitialWorkset(verticesWithId);
    iteration.setMaximumNumberOfIterations(maxIterations);
   
    // create DataSourceContract for the edges
    FileDataSource edges = new FileDataSource(new CsvInputFormat(' ', LongValue.class, LongValue.class), edgeInput, "Edges");

    // create CrossOperator for distance computation
    JoinOperator joinWithNeighbors = JoinOperator.builder(new NeighborWithComponentIDJoin(), LongValue.class, 0, 0)
        .input1(iteration.getWorkset())
        .input2(edges)
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
136
    return toParameterList(config1);
  }
 
  static Plan getTestPlanPlan(int numSubTasks, String input, String output) {
   
    FileDataSource source = new FileDataSource(new TextInputFormat(), input, "Input Lines");
    source.setParameter(TextInputFormat.CHARSET_NAME, "ASCII");
    MapOperator mapper = MapOperator.builder(new TokenizeLine())
      .input(source)
      .name("Tokenize Lines")
      .build();
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
View Full Code Here
TOP

Related Classes of eu.stratosphere.api.java.record.operators.FileDataSource

  • eu.stratosphere.pact.compiler.AdditionalOperatorsTest
  • eu.stratosphere.pact.compiler.BranchingPlansCompilerTest
  • eu.stratosphere.pact.compiler.DOPChangeTest
  • eu.stratosphere.pact.compiler.GroupOrderTest
  • eu.stratosphere.pact.compiler.HardPlansCompilationTest
  • eu.stratosphere.pact.compiler.ReduceAllTest
  • eu.stratosphere.pact.compiler.UnionPropertyPropagationTest
  • eu.stratosphere.pact.compiler.WorksetIterationsRecordApiCompilerTest
  • eu.stratosphere.test.accumulators.AccumulatorITCase
  • eu.stratosphere.test.accumulators.AccumulatorIterativeITCase
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.