Package com.google.appengine.tools.mapreduce.impl

Examples of com.google.appengine.tools.mapreduce.impl.BaseContext

75
76
77
78
79
80
81
82
83
84
85
        queue = "default";
      }
      settings = new MapReduceSettings.Builder().setWorkerQueueName(queue).build();
    }
    String jobId = getJobKey().getName();
    Context context = new BaseContext(jobId);
    Input<I> input = specification.getInput();
    input.setContext(context);
    List<? extends InputReader<I>> readers;
    try {
      readers = input.createReaders();
View Full Code Here
211
212
213
214
215
216
217
218
219
220
221
     * The format for how the data is written out is defined by {@link GoogleCloudStorageSortOutput}
     */
    @Override
    public Value<MapReduceResult<FilesByShard>> run(MapReduceResult<FilesByShard> mapResult) {

      Context context = new BaseContext(mrJobId);
      int mapShards = findMaxFilesPerShard(mapResult.getOutputResult());
      int reduceShards = mrSpec.getNumReducers();
      FilesByShard filesByShard = mapResult.getOutputResult();
      filesByShard.splitShards(Math.max(mapShards, reduceShards));
      GoogleCloudStorageSortInput input = new GoogleCloudStorageSortInput(filesByShard);
View Full Code Here
301
302
303
304
305
306
307
308
309
310
311
     * method will invoke itself recursively.
     */
    @Override
    public Value<MapReduceResult<FilesByShard>> run(MapReduceResult<FilesByShard> priorResult) {

      Context context = new BaseContext(mrJobId);
      FilesByShard sortFiles = priorResult.getOutputResult();
      int maxFilesPerShard = findMaxFilesPerShard(sortFiles);
      if (maxFilesPerShard <= settings.getMergeFanin()) {
        return immediate(priorResult);
      }
View Full Code Here
402
403
404
405
406
407
408
409
410
411
412
     * ordered input.
     * The way the data is read in is defined by {@link GoogleCloudStorageReduceInput}
     */
    @Override
    public Value<MapReduceResult<R>> run(MapReduceResult<FilesByShard> mergeResult) {
      Context context = new BaseContext(mrJobId);
      Output<O, R> output = mrSpec.getOutput();
      output.setContext(context);
      GoogleCloudStorageReduceInput<K, V> input = new GoogleCloudStorageReduceInput<>(
          mergeResult.getOutputResult(), mrSpec.getKeyMarshaller(), mrSpec.getValueMarshaller());
      ((Input<?>) input).setContext(context);
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
     *
     * @returns A future containing the FilesByShard for the sortJob
     */
    @Override
    public Value<MapReduceResult<FilesByShard>> run() {
      Context context = new BaseContext(mrJobId);
      Input<I> input = mrSpec.getInput();
      input.setContext(context);
      List<? extends InputReader<I>> readers;
      try {
        readers = input.createReaders();
View Full Code Here
TOP

Related Classes of com.google.appengine.tools.mapreduce.impl.BaseContext

  • com.google.appengine.tools.mapreduce.MapJob
  • com.google.appengine.tools.mapreduce.MapReduceJob$MapJob
  • com.google.appengine.tools.mapreduce.MapReduceJob$MergeJob
  • com.google.appengine.tools.mapreduce.MapReduceJob$ReduceJob
  • com.google.appengine.tools.mapreduce.MapReduceJob$SortJob
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.