Package com.google.appengine.tools.mapreduce

Examples of com.google.appengine.tools.mapreduce.MapReduceSettings$Builder


  public void setUp() {
    helper.setUp();
  }

  public void testDefaultSettings() {
    MapReduceSettings mrSettings = new MapReduceSettings.Builder().build();
    assertNull(mrSettings.getBackend());
    assertEquals(DEFAULT_BASE_URL, mrSettings.getBaseUrl());
    assertEquals("app_default_bucket", mrSettings.getBucketName());
    assertEquals(DEFAULT_MAP_FANOUT, mrSettings.getMapFanout());
    assertEquals(DEFAULT_SHARD_RETREIES, mrSettings.getMaxShardRetries());
    assertEquals(DEFAULT_SLICE_RETREIES, mrSettings.getMaxSliceRetries());
    assertNull(mrSettings.getMaxSortMemory());
    assertEquals(DEFAULT_MERGE_FANIN, mrSettings.getMergeFanin());
    assertEquals(DEFAULT_MILLIS_PER_SLICE, mrSettings.getMillisPerSlice());
    assertEquals(null, mrSettings.getModule());
    assertEquals(DEFAULT_SORT_BATCH_PER_EMIT_BYTES, mrSettings.getSortBatchPerEmitBytes());
    assertEquals(DEFAULT_SORT_READ_TIME_MILLIS, mrSettings.getSortReadTimeMillis());
    assertNull(mrSettings.getWorkerQueueName());
  }
View Full Code Here


    }
    builder = builder.setSortReadTimeMillis(6);
    builder = builder.setWorkerQueueName("queue1");


    MapReduceSettings mrSettings = builder.build();
    assertEquals("b1", mrSettings.getBackend());
    assertNull(mrSettings.getModule());
    assertEquals("bucket", mrSettings.getBucketName());
    assertEquals("base-url", mrSettings.getBaseUrl());
    assertEquals(3, mrSettings.getMapFanout());
    assertEquals(1, mrSettings.getMaxShardRetries());
    assertEquals(0, mrSettings.getMaxSliceRetries());
    assertEquals(10L, (long) mrSettings.getMaxSortMemory());
    assertEquals(4, mrSettings.getMergeFanin());
    assertEquals(10, mrSettings.getMillisPerSlice());
    assertEquals(5, mrSettings.getSortBatchPerEmitBytes());
    assertEquals(6, mrSettings.getSortReadTimeMillis());
    assertEquals("queue1", mrSettings.getWorkerQueueName());

    builder = new MapReduceSettings.Builder().setModule("m1");
    try {
      builder.setBackend("b").build();
      fail("Expected exception to be thrown");
    } catch (IllegalArgumentException ex) {
      // expected
      builder.setBackend(null);
    }
    mrSettings = builder.build();
    assertNull(mrSettings.getBackend());
    assertEquals("m1", mrSettings.getModule());
  }
View Full Code Here

    @Override
    public Value<Void> run() throws Exception {
      MapReduceSpecification<Long, Integer, Integer, ArrayList<Integer>, GoogleCloudStorageFileSet>
      spec = createMapReduceSpec(bucket, start, limit, shards);
      MapReduceSettings settings = getSettings(bucket, null, null);
      // [START start_as_pipeline]
      FutureValue<MapReduceResult<GoogleCloudStorageFileSet>> mapReduceResult =
          futureCall(new MapReduceJob<>(spec, settings));
      // [END start_as_pipeline]
      return futureCall(new LogFileNamesJob(), mapReduceResult);
View Full Code Here

    long limit = getLongParam(req, "limit", 3 * 1000 * 1000);
    int shards = Math.max(1, Math.min(100, Ints.saturatedCast(getLongParam(req, "shards", 10))));
    // [START start_mapreduce]
    MapReduceSpecification<Long, Integer, Integer, ArrayList<Integer>, GoogleCloudStorageFileSet>
        mapReduceSpec = createMapReduceSpec(bucket, start, limit, shards);
    MapReduceSettings settings = getSettings(bucket, queue, module);
    // [START startMapReduceJob]
    String id = MapReduceJob.start(mapReduceSpec, settings);
    // [END startMapReduceJob]
    // [END start_mapreduce]
    resp.sendRedirect("/_ah/pipeline/status.html?root=" + id);
View Full Code Here

  // [END createMapReduceSpec]

  // [START getSettings]
  public static MapReduceSettings getSettings(String bucket, String queue, String module) {
    // [START mapReduceSettings]
    MapReduceSettings settings = new MapReduceSettings.Builder()
        .setBucketName(bucket)
        .setWorkerQueueName(queue)
        .setModule(module) // if queue is null will use the current queue or "default" if none
        .build();
    // [END mapReduceSettings]
View Full Code Here

        new MapReduceSpecification.Builder<>(input, mapper, reducer, output)
            .setKeyMarshaller(Marshallers.getStringMarshaller())
            .setValueMarshaller(Marshallers.getStringMarshaller()).setNumReducers(reduceShardCount)
            .setJobName("history").build();

    MapReduceSettings mrsettings = new MapReduceSettings.Builder().build();
    return MapReduceJob.start(mrspecs, mrsettings);
  }
View Full Code Here

    }
    writeResponse(resp);
  }

  private MapReduceSettings getSettings() {
    MapReduceSettings settings = new MapReduceSettings()
        .setWorkerQueueName("mapreduce-workers")
        .setControllerQueueName("default");
    if (USE_BACKENDS) {
      settings.setBackend("worker");
    }
    return settings;
  }
View Full Code Here

TOP

Related Classes of com.google.appengine.tools.mapreduce.MapReduceSettings$Builder

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.