Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.ReduceOperator

901
902
903
904
905
906
907
908
909
910
911
    MapOperator mappedSource = MapOperator.builder(IdentityMap.class).
        input(source).
        name("Identity mapped source").
        build();

    ReduceOperator reducedSource = ReduceOperator.builder(IdentityReduce.class).
        input(source).
        name("Identity reduce source").
        build();

    BulkIteration iteration = new BulkIteration("Loop");
View Full Code Here
199
200
201
202
203
204
205
206
207
208
209
210
211
212
    edges.setParameter(EdgeInputFormat.ID_DELIMITER_CHAR, delimiter);
   
    MapOperator projectEdge = MapOperator.builder(new ProjectEdge())
      .input(edges).name("Project Edge").build();
   
    ReduceOperator edgeCounter = ReduceOperator.builder(new CountEdges(), IntValue.class, 0)
      .input(projectEdge).name("Count Edges for Vertex").build();
   
    ReduceOperator countJoiner = ReduceOperator.builder(new JoinCountsAndUniquify())
      .keyField(IntValue.class, 0)
      .keyField(IntValue.class, 1)
      .input(edgeCounter)
      .name("Join Counts")
      .build();
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
    source.setParameter(TextInputFormat.CHARSET_NAME, "ASCII");
    MapOperator mapper = MapOperator.builder(new TokenizeLine())
      .input(source)
      .name("Tokenize Lines")
      .build();
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
      .input(mapper)
      .name("Count Words")
      .build();
    @SuppressWarnings("unchecked")
    FileDataSink out = new FileDataSink(new CsvOutputFormat("\n"," ", StringValue.class, IntValue.class), output, reducer, "Word Counts");
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
        .input2(iteration.getPartialSolution())
        .name("Compute Distances")
        .build();

    // create ReduceOperator for finding the nearest cluster centers
    ReduceOperator findNearestClusterCenters = ReduceOperator.builder(new FindNearestCenter(), IntValue.class, 0)
        .input(computeDistance)
        .name("Find Nearest Centers")
        .build();

    // create ReduceOperator for computing new cluster positions
    ReduceOperator recomputeClusterCenter = ReduceOperator.builder(new RecomputeClusterCenter(), IntValue.class, 0)
        .input(findNearestClusterCenters)
        .name("Recompute Center Positions")
        .build();
    iteration.setNextPartialSolution(recomputeClusterCenter);
   
    // create DataSourceContract for data point input
    FileDataSource dataPoints2 = new FileDataSource(new PointInFormat(), dataPointInput, "Data Points 2");
   
    // compute distance of points to final clusters
    CrossOperator computeFinalDistance = CrossOperator.builder(new ComputeDistance())
        .input1(dataPoints2)
        .input2(iteration)
        .name("Compute Final Distances")
        .build();

    // find nearest final cluster for point
    ReduceOperator findNearestFinalCluster = ReduceOperator.builder(new FindNearestCenter(), IntValue.class, 0)
        .input(computeFinalDistance)
        .name("Find Nearest Final Centers")
        .build();

    // create DataSinkContract for writing the new cluster positions
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
   
    @SuppressWarnings("unchecked")
    CsvInputFormat format = new CsvInputFormat(',', IntValue.class, IntValue.class);
    FileDataSource source = new FileDataSource(format, this.textPath, "Source");
   
    ReduceOperator reducer = ReduceOperator.builder(CheckingReducer.class)
      .keyField(IntValue.class, 0)
      .input(source)
      .name("Ordered Reducer")
      .build();
    reducer.setGroupOrder(new Ordering(1, IntValue.class, Order.ASCENDING));
   
    FileDataSink sink = new FileDataSink(CsvOutputFormat.class, this.resultPath, reducer, "Sink");
    CsvOutputFormat.configureRecordFormat(sink)
      .recordDelimiter('\n')
      .fieldDelimiter(',')
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
    @SuppressWarnings("unchecked")
    CsvInputFormat format = new CsvInputFormat(' ', IntValue.class, IntValue.class);
    FileDataSource input = new FileDataSource(format, dataInput, "Input");
   
    // create the reduce contract and sets the key to the first field
    ReduceOperator sorter = ReduceOperator.builder(new IdentityReducer(), IntValue.class, 0)
      .input(input)
      .name("Reducer")
      .build();
    // sets the group sorting to the second field
    sorter.setGroupOrder(new Ordering(1, IntValue.class, Order.ASCENDING));

    // create and configure the output format
    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, sorter, "Sorted Output");
    CsvOutputFormat.configureRecordFormat(out)
      .recordDelimiter('\n')
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
      .input(dataPoints)
      .name("Find Nearest Centers")
      .build();

    // computing the new cluster positions
    ReduceOperator recomputeClusterCenter = ReduceOperator.builder(new RecomputeClusterCenter(), IntValue.class, 0)
      .input(findNearestClusterCenters)
      .name("Recompute Center Positions")
      .build();
   
    iter.setNextPartialSolution(recomputeClusterCenter);
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
        .input2(edges)
        .name("Join Candidate Id With Neighbor")
        .build();

    // find for each neighbor the smallest of all candidates
    ReduceOperator minCandidateId = ReduceOperator.builder(new MinimumComponentIDReduce(), LongValue.class, 0)
        .input(joinWithNeighbors)
        .name("Find Minimum Candidate Id")
        .build();
   
    // join candidates with the solution set and update if the candidate component-id is smaller
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
   
    BulkIteration iteration = new BulkIteration("Loop");
    iteration.setInput(initialInput);
    iteration.setMaximumNumberOfIterations(NUM_ITERATIONS);

    ReduceOperator sumReduce = ReduceOperator.builder(new SumReducer())
        .input(iteration.getPartialSolution())
        .name("Compute sum (Reduce)")
        .build();
   
    iteration.setNextPartialSolution(sumReduce);
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
        .input1(iteration.getWorkset())
        .input2(dependencySet)
        .name("calculate dependencies")
        .build();
   
    ReduceOperator updateRanks = ReduceOperator.builder(UpdateRankReduceDelta.class, LongValue.class, 0)
        .input(dependenciesMatch)
        .name("update ranks")
        .build();
   
    JoinOperator oldRankComparison = JoinOperator.builder(RankComparisonMatch.class, LongValue.class, 0, 0)
View Full Code Here
TOP

Related Classes of eu.stratosphere.api.java.record.operators.ReduceOperator

  • eu.stratosphere.api.java.record.io.avro.example.ReflectiveAvroTypeExample
  • eu.stratosphere.hadoopcompatibility.example.WordCount
  • eu.stratosphere.hadoopcompatibility.example.WordCountWithHadoopOutputFormat
  • eu.stratosphere.pact.compiler.BranchingPlansCompilerTest
  • eu.stratosphere.pact.compiler.DOPChangeTest
  • eu.stratosphere.pact.compiler.GroupOrderTest
  • eu.stratosphere.pact.compiler.HardPlansCompilationTest
  • eu.stratosphere.pact.compiler.ReduceAllTest
  • eu.stratosphere.pact.compiler.UnionPropertyPropagationTest
  • eu.stratosphere.pact.compiler.WorksetIterationsRecordApiCompilerTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.