Package com.google.appengine.tools.mapreduce.impl

Examples of com.google.appengine.tools.mapreduce.impl.HashingSharder

38
39
40
41
42
43
44
45
46
47
48
  }

  public void testCreatorCalled() throws IOException {
    int numShards = 10;
    TestShardingOutputWriter writer = new TestShardingOutputWriter(
        Marshallers.getIntegerMarshaller(), new HashingSharder(numShards));
    writer.beginShard();
    writer.beginSlice();
    for (int i = 0; i < numShards * 10; i++) {
      writer.write(new KeyValue<>(i, i));
    }
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
    final AtomicInteger shardEnds = new AtomicInteger(0);
    final AtomicInteger sliceBegins = new AtomicInteger(0);
    final AtomicInteger sliceEnds = new AtomicInteger(0);
    ShardingOutputWriter<Integer, Integer, OutputWriter<KeyValue<Integer, Integer>>> writer =
        new ShardingOutputWriter<Integer, Integer, OutputWriter<KeyValue<Integer, Integer>>>(
            Marshallers.getIntegerMarshaller(), new HashingSharder(numShards)) {
          private static final long serialVersionUID = 1L;

          @Override
          public OutputWriter<KeyValue<Integer, Integer>> createWriter(int number) {
            return new InMemoryOutputWriter<KeyValue<Integer, Integer>>() {
View Full Code Here
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
        List<ArrayListMultimap<String, Long>> expectedOutput = Lists.newArrayList();
        for (int i = 0; i < 5; i++) {
          expectedOutput.add(ArrayListMultimap.<String, Long>create());
        }
        Marshaller<String> marshaller = Marshallers.getStringMarshaller();
        HashingSharder sharder = new HashingSharder(5);
        for (long l = -10000; l < 10000; l++) {
          String mod37 = String.valueOf(Math.abs(l) % 37);
          expectedOutput.get(sharder.getShardForKey(marshaller.toBytes(mod37))).put(mod37, l);
        }
        for (int i = 0; i < 5; i++) {
          assertEquals(expectedOutput.get(i).keySet().size(), actualOutput.get(i).size());
          for (KeyValue<String, List<Long>> actual : actualOutput.get(i)) {
            List<Long> value = new ArrayList<>(actual.getValue());
View Full Code Here
221
222
223
224
225
226
227
228
229
230
231
      GoogleCloudStorageSortInput input = new GoogleCloudStorageSortInput(filesByShard);
      ((Input<?>) input).setContext(context);
      List<? extends InputReader<KeyValue<ByteBuffer, ByteBuffer>>> readers = input.createReaders();
      Output<KeyValue<ByteBuffer, List<ByteBuffer>>, FilesByShard> output =
          new GoogleCloudStorageSortOutput(settings.getBucketName(), mrJobId,
              new HashingSharder(reduceShards));
      output.setContext(context);

      List<? extends OutputWriter<KeyValue<ByteBuffer, List<ByteBuffer>>>> writers =
          output.createWriters(readers.size());
      Preconditions.checkState(readers.size() == writers.size(), "%s: %s readers, %s writers",
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
      Output<KeyValue<K, V>, FilesByShard> output = new GoogleCloudStorageMapOutput<>(
              settings.getBucketName(),
              mrJobId,
              mrSpec.getKeyMarshaller(),
              mrSpec.getValueMarshaller(),
              new HashingSharder(getNumOutputFiles(readers.size())));
      output.setContext(context);

      List<? extends OutputWriter<KeyValue<K, V>>> writers = output.createWriters(readers.size());
      Preconditions.checkState(readers.size() == writers.size(), "%s: %s readers, %s writers",
          shardedJobId, readers.size(), writers.size());
View Full Code Here
TOP

Related Classes of com.google.appengine.tools.mapreduce.impl.HashingSharder

  • com.google.appengine.tools.mapreduce.EndToEndTest
  • com.google.appengine.tools.mapreduce.MapReduceJob$MapJob
  • com.google.appengine.tools.mapreduce.MapReduceJob$SortJob
  • com.google.appengine.tools.mapreduce.outputs.ShardingOutputWriterTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.