Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.FileDataSink

209
210
211
212
213
214
215
216
217
218
219
      .keyField(IntValue.class, 1)
      .input(edgeCounter)
      .name("Join Counts")
      .build();

    FileDataSink triangles = new FileDataSink(new EdgeWithDegreesOutputFormat(), output, countJoiner, "Unique Edges With Degrees");

    Plan p = new Plan(triangles, "Normalize Edges and compute Vertex Degrees");
    p.setDefaultParallelism(numSubTasks);
    return p;
  }
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
   
    iteration.setNextWorkset(minAndUpdate);
    iteration.setSolutionSetDelta(minAndUpdate);

    // create DataSinkContract for writing the new cluster positions
    FileDataSink result = new FileDataSink(new CsvOutputFormat(), output, iteration, "Result");
    CsvOutputFormat.configureRecordFormat(result)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(LongValue.class, 0)
      .field(LongValue.class, 1);
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
      .build();
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
      .input(mapper)
      .name("Count Words")
      .build();
    @SuppressWarnings("unchecked")
    FileDataSink out = new FileDataSink(new CsvOutputFormat("\n"," ", StringValue.class, IntValue.class), output, reducer, "Word Counts");

    Plan plan = new Plan(out, "WordCount Example");
    plan.setDefaultParallelism(numSubTasks);
   
    return plan;
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
104
105
        .input(computeFinalDistance)
        .name("Find Nearest Final Centers")
        .build();

    // create DataSinkContract for writing the new cluster positions
    FileDataSink finalClusters = new FileDataSink(new PointOutFormat(), output+"/centers", iteration, "Cluster Positions");

    // write assigned clusters
    FileDataSink clusterAssignments = new FileDataSink(new PointOutFormat(), output+"/points", findNearestFinalCluster, "Cluster Assignments");
   
    List<FileDataSink> sinks = new ArrayList<FileDataSink>();
    sinks.add(finalClusters);
    sinks.add(clusterAssignments);
   
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
      .input(source)
      .name("Ordered Reducer")
      .build();
    reducer.setGroupOrder(new Ordering(1, IntValue.class, Order.ASCENDING));
   
    FileDataSink sink = new FileDataSink(CsvOutputFormat.class, this.resultPath, reducer, "Sink");
    CsvOutputFormat.configureRecordFormat(sink)
      .recordDelimiter('\n')
      .fieldDelimiter(',')
      .field(IntValue.class, 0)
      .field(IntValue.class, 1);
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
      .build();
    // sets the group sorting to the second field
    sorter.setGroupOrder(new Ordering(1, IntValue.class, Order.ASCENDING));

    // create and configure the output format
    FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, sorter, "Sorted Output");
    CsvOutputFormat.configureRecordFormat(out)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(IntValue.class, 0)
      .field(IntValue.class, 1);
View Full Code Here
174
175
176
177
178
179
180
181
182
183
184
   
    iteration.setNextWorkset(updateComponentId);
    iteration.setSolutionSetDelta(updateComponentId);

    // sink is the iteration result
    FileDataSink result = new FileDataSink(new CsvOutputFormat(), output, iteration, "Result");
    CsvOutputFormat.configureRecordFormat(result)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(LongValue.class, 0)
      .field(LongValue.class, 1);
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
    iter.setNextPartialSolution(recomputeClusterCenter);

    // ---------------------- End K-Means Loop ---------------------
   
    // create DataSinkContract for writing the new cluster positions
    FileDataSink newClusterPoints = new FileDataSink(new PointOutFormat(), output, iter, "New Center Positions");

    Plan plan = new Plan(newClusterPoints, "K-Means");
    plan.setDefaultParallelism(degreeOfParallelism);
    return plan;
  }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
    final FileDataSource source =
        new FileDataSource(new TeraInputFormat(), input, "Data Source");
    source.setDegreeOfParallelism(numSubTasks);

    // This task writes the sorted data back to disk
    final FileDataSink sink =
        new FileDataSink(new TeraOutputFormat(), output, "Data Sink");
    sink.setDegreeOfParallelism(numSubTasks);
    sink.setGlobalOrder(new Ordering(0, TeraKey.class, Order.ASCENDING), new TeraDistribution());

    sink.setInput(source);

    return new Plan(sink, "TeraSort");
  }
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
90
91
   
    ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
    reduce2.setDegreeOfParallelism(degOfPar * 2);
    reduce2.setInput(map2);
   
    FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
    sink.setDegreeOfParallelism(degOfPar * 2);
    sink.setInput(reduce2);
   
    Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
   
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);
View Full Code Here
TOP

Related Classes of eu.stratosphere.api.java.record.operators.FileDataSink

  • eu.stratosphere.addons.hbase.example.HBaseReadExample
  • eu.stratosphere.api.java.typeutils.RecordTypeInfo
  • eu.stratosphere.hadoopcompatibility.example.WordCount
  • eu.stratosphere.pact.compiler.AdditionalOperatorsTest
  • eu.stratosphere.pact.compiler.BranchingPlansCompilerTest
  • eu.stratosphere.pact.compiler.DOPChangeTest
  • eu.stratosphere.pact.compiler.GroupOrderTest
  • eu.stratosphere.pact.compiler.HardPlansCompilationTest
  • eu.stratosphere.pact.compiler.ReduceAllTest
  • eu.stratosphere.pact.compiler.UnionPropertyPropagationTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.