Examples of GoogleCloudStorageFileSet


Examples of com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet

      out.beginSlice();
      out.write(ByteBuffer.wrap(SMALL_CONTENT));
      out.endSlice();
      out.endShard();
    }
    GoogleCloudStorageFileSet files = creator.finish(writers);
    assertEquals(NUM_SHARDS, files.getNumFiles());
    for (int i = 0; i < NUM_SHARDS; i++) {
      GcsFileMetadata metadata = gcsService.getMetadata(files.getFile(i));
      assertNotNull(metadata);
      assertEquals(SMALL_CONTENT.length, metadata.getLength());
      assertEquals(MIME_TYPE, metadata.getOptions().getMimeType());
    }
  }
View Full Code Here

Examples of com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet

      out.beginSlice();
      out.write(ByteBuffer.wrap(content));
      out.endSlice();
      out.endShard();
    }
    GoogleCloudStorageFileSet files = creator.finish(writers);
    assertEquals(NUM_SHARDS, files.getNumFiles());
    ByteBuffer expectedContent = ByteBuffer.allocate(content.length * 2);
    expectedContent.put(content);
    expectedContent.put(content);
    for (int i = 0; i < NUM_SHARDS; i++) {
      expectedContent.rewind();
      ByteBuffer actualContent = ByteBuffer.allocate(content.length * 2 + 1);
      GcsFileMetadata metadata = gcsService.getMetadata(files.getFile(i));
      assertNotNull(metadata);
      assertEquals(expectedContent.capacity(), metadata.getLength());
      assertEquals(MIME_TYPE, metadata.getOptions().getMimeType());
      try (ReadableByteChannel readChannel = gcsService.openReadChannel(files.getFile(i), 0)) {
        int read = readChannel.read(actualContent);
        assertEquals(read, content.length * 2);
        actualContent.limit(actualContent.position());
        actualContent.rewind();
        assertEquals(expectedContent, actualContent);
View Full Code Here

Examples of com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet

      w.write(ByteBuffer.wrap(new byte[9]));
      w.endSlice();
      w.endShard();
      finished.add(w);
    }
    GoogleCloudStorageFileSet filesWritten = segmenter.finish(finished);
    assertEquals(15, filesWritten.getNumFiles());
    for (int i = 0; i < filesWritten.getNumFiles(); i++) {
      GcsFileMetadata metadata = gcsService.getMetadata(filesWritten.getFile(i));
      assertNotNull(metadata);
      assertEquals(MIME_TYPE, metadata.getOptions().getMimeType());
    }
  }
View Full Code Here

Examples of com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet

    int countFiles = 0;
    for (OutputWriter<ByteBuffer> w : writers) {
      writeMultipleValues(w, 3, 9);
      countFiles += 3;
    }
    GoogleCloudStorageFileSet filesWritten = segmenter.finish(writers);
    assertEquals(countFiles, filesWritten.getNumFiles());
    for (int i = 0; i < filesWritten.getNumFiles(); i++) {
      GcsFileMetadata metadata = gcsService.getMetadata(filesWritten.getFile(i));
      assertNotNull(metadata);
      assertEquals(MIME_TYPE, metadata.getOptions().getMimeType());
    }
  }
View Full Code Here

Examples of com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet

  public void testAdd() {
    FilesByShard fbs = new FilesByShard(10, "Foo");
    fbs.addFileToShard(1, "Bar");
    fbs.addFileToShard(1, "Baz");
    fbs.addFileToShard(1, "Bat");
    GoogleCloudStorageFileSet forShard = fbs.getFilesForShard(1);
    List<GcsFilename> files = forShard.getFiles();
    assertEquals(3, files.size());
    assertEquals("Foo", files.get(0).getBucketName());
    assertEquals("Bar", files.get(0).getObjectName());
    assertEquals("Baz", files.get(1).getObjectName());
    assertEquals("Bat", files.get(2).getObjectName());
View Full Code Here

Examples of com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet

  }

  public void testAddAll() {
    FilesByShard fbs = new FilesByShard(10, "Foo");
    fbs.addFilesToShard(1, Arrays.asList("Bar", "Baz", "Bat"));
    GoogleCloudStorageFileSet forShard = fbs.getFilesForShard(1);
    List<GcsFilename> files = forShard.getFiles();
    assertEquals(3, files.size());
    assertEquals("Foo", files.get(0).getBucketName());
    assertEquals("Bar", files.get(0).getObjectName());
    assertEquals("Baz", files.get(1).getObjectName());
    assertEquals("Bat", files.get(2).getObjectName());
View Full Code Here

Examples of com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet

    assertEquals("Bat", files.get(2).getObjectName());
  }

  public void testEmptyShards() {
    FilesByShard fbs = new FilesByShard(10, "Foo");
    GoogleCloudStorageFileSet forShard = fbs.getFilesForShard(1);
    List<GcsFilename> files = forShard.getFiles();
    assertEquals(0, files.size());
  }
View Full Code Here

Examples of com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet

    List<String> out = Lists.newArrayList();
    for (OutputWriter<ByteBuffer> w : writers) {
      GoogleCloudStorageFileOutputWriter writer = (GoogleCloudStorageFileOutputWriter) w;
      out.add(writer.getFile().getObjectName());
    }
    return new GoogleCloudStorageFileSet(bucket, out);
  }
View Full Code Here

Examples of com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet

  }

  public GoogleCloudStorageFileSet getFilesForShard(int shardNumber) {
    Preconditions.checkArgument(shardNumber < shardCount);
    List<String> fileNames = allFiles.get(shardNumber);
    return new GoogleCloudStorageFileSet(getBucket(), fileNames);
  }
View Full Code Here

Examples of com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet

        ImmutableList.builder();
    for (int shard = 0; shard < filesByShard.getShardCount(); shard++) {
      List<InputReader<KeyValue<ByteBuffer, Iterator<ByteBuffer>>>> readers = new ArrayList<>();
      for (List<String> group : Lists.partition(filesByShard.getFilesForShard(shard).getFileNames(),
          mergeFanin)) {
        GoogleCloudStorageFileSet fileSet =
            new GoogleCloudStorageFileSet(filesByShard.getBucket(), group);
        readers.add(createReaderForShard(marshaller, fileSet));
      }
      result.add(new ConcatenatingInputReader<>(readers));
    }
    return result.build();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.