cleanupBucketForTest("testMultipartUtilities");
}
}
public void testMultipartUploads() throws Exception {
RestS3Service service = (RestS3Service) getStorageService(getCredentials());
StorageBucket bucket = createBucketForTest("testMultipartUploads");
String bucketName = bucket.getName();
try {
// Check stripping of double-quote characters from etag
MultipartPart testEtagSanitized = new MultipartPart(
1, new Date(), "\"fakeEtagWithDoubleQuotes\"", 0l);
assertEquals("fakeEtagWithDoubleQuotes", testEtagSanitized.getEtag());
// Create 5MB of test data
int fiveMB = 5 * 1024 * 1024;
byte[] fiveMBTestData = new byte[fiveMB];
for (int offset = 0; offset < fiveMBTestData.length; offset++) {
fiveMBTestData[offset] = (byte) (offset % 256);
}
// Define name and String metadata values for multipart upload object
String objectKey = "multipart-object.txt";
Map<String, Object> metadata = new HashMap<String, Object>();
metadata.put("test-md-value", "testing, testing, 123");
metadata.put("test-timestamp-value", System.currentTimeMillis());
// Start a multipart upload
MultipartUpload testMultipartUpload = service.multipartStartUpload(
bucketName, objectKey, metadata,
AccessControlList.REST_CANNED_AUTHENTICATED_READ, null);
assertEquals(bucketName, testMultipartUpload.getBucketName());
assertEquals(objectKey, testMultipartUpload.getObjectKey());
// List all ongoing multipart uploads
List<MultipartUpload> uploads = service.multipartListUploads(bucketName);
assertTrue("Expected at least one ongoing upload", uploads.size() >= 1);
// Confirm our newly-created multipart upload is present in listing
boolean foundNewUpload = false;
for (MultipartUpload upload: uploads) {
if (upload.getUploadId().equals(testMultipartUpload.getUploadId())) {
foundNewUpload = true;
}
}
assertTrue("Expected to find the new upload in listing", foundNewUpload);
// Start a second, encrypted multipart upload
S3Object encryptedMultiPartObject = new S3Object(objectKey + "2");
encryptedMultiPartObject.setServerSideEncryptionAlgorithm(
S3Object.SERVER_SIDE_ENCRYPTION__AES256);
MultipartUpload testMultipartUpload2 =
service.multipartStartUpload(bucketName, encryptedMultiPartObject);
assertEquals("AES256",
testMultipartUpload2.getMetadata().get("x-amz-server-side-encryption"));
// List multipart uploads with markers -- Find second upload only
uploads = service.multipartListUploads(bucketName,
"multipart-object.txt",
testMultipartUpload.getUploadId(),
10);
assertEquals(1, uploads.size());
assertEquals(objectKey + "2", uploads.get(0).getObjectKey());
// List multipart uploads with prefix/delimiter constraints
MultipartUpload testMultipartUpload3 =
service.multipartStartUpload(bucketName, objectKey + "/delimited", metadata);
MultipartUploadChunk chunk = service.multipartListUploadsChunked(bucketName,
"multipart-object", // prefix
null, // delimiter
null, null, 1000, true);
assertEquals("multipart-object", chunk.getPrefix());
assertEquals(null, chunk.getDelimiter());
assertEquals(3, chunk.getUploads().length);
chunk = service.multipartListUploadsChunked(bucketName,
"multipart-object.txt2", // prefix
null, // delimiter
null, null, 1000, true);
assertEquals("multipart-object.txt2", chunk.getPrefix());
assertEquals(null, chunk.getDelimiter());
assertEquals(1, chunk.getUploads().length);
chunk = service.multipartListUploadsChunked(bucketName,
"multipart-object", // prefix
"/", // delimiter
null, null, 1000, true);
assertEquals("multipart-object", chunk.getPrefix());
assertEquals("/", chunk.getDelimiter());
assertEquals(2, chunk.getUploads().length);
assertEquals(1, chunk.getCommonPrefixes().length);
assertEquals("multipart-object.txt/", chunk.getCommonPrefixes()[0]);
chunk = service.multipartListUploadsChunked(bucketName,
"multipart-object", // prefix
null, // delimiter
null, null,
1, // Max number of uploads to return per LIST request
false // Do *not* complete listing, just get first chunk
);
assertEquals(1, chunk.getUploads().length);
assertEquals(0, chunk.getCommonPrefixes().length);
chunk = service.multipartListUploadsChunked(bucketName,
"multipart-object", // prefix
null, // delimiter
null, null,
1, // Max number of uploads to return per LIST request
true // *Do* complete listing, 1 item at a time
);
assertEquals(3, chunk.getUploads().length);
assertEquals(0, chunk.getCommonPrefixes().length);
// Delete incomplete/unwanted multipart uploads
service.multipartAbortUpload(testMultipartUpload2);
service.multipartAbortUpload(testMultipartUpload3);
// Ensure the incomplete multipart upload has been deleted
uploads = service.multipartListUploads(bucketName);
for (MultipartUpload upload: uploads) {
if (upload.getUploadId().equals(testMultipartUpload2.getUploadId()))
{
fail("Expected multipart upload " + upload.getUploadId()
+ " to be deleted");
}
}
int partNumber = 0;
// Upload a first part, must be 5MB+
S3Object partObject = new S3Object(
testMultipartUpload.getObjectKey(), fiveMBTestData);
MultipartPart uploadedPart = service.multipartUploadPart(
testMultipartUpload, ++partNumber, partObject);
assertEquals(uploadedPart.getPartNumber().longValue(), partNumber);
assertEquals(uploadedPart.getEtag(), partObject.getETag());
assertEquals(uploadedPart.getSize().longValue(), partObject.getContentLength());
// List multipart parts that have been received by the service
List<MultipartPart> listedParts = service.multipartListParts(testMultipartUpload);
assertEquals(listedParts.size(), 1);
assertEquals(listedParts.get(0).getSize().longValue(), partObject.getContentLength());
// Upload a second part by copying an object already in S3, must be >= 5 MB
S3Object objectToCopy = service.putObject(bucketName,
new S3Object("objectToCopy.txt", fiveMBTestData));
MultipartPart copiedPart = service.multipartUploadPartCopy(testMultipartUpload,
++partNumber, bucketName, objectToCopy.getKey());
assertEquals(copiedPart.getPartNumber().longValue(), partNumber);
assertEquals(copiedPart.getEtag(), objectToCopy.getETag());
// Note: result part from copy operation does *not* include correct part size, due
// to lack of this info in the CopyPartResult XML response.
// assertEquals(copiedPart.getSize().longValue(), partObject.getContentLength());
// List multipart parts that have been received by the service
listedParts = service.multipartListParts(testMultipartUpload);
assertEquals(listedParts.size(), 2);
assertEquals(listedParts.get(1).getSize().longValue(), objectToCopy.getContentLength());
// TODO Test multipart upload copy with version ID
// TODO Test multipart upload copy with byte range (need object >= 5 GB !)
// TODO Test multipart upload copy with ETag (mis)match test
// TODO Test multipart upload copy with (un)modified since test
// Upload a third and final part, can be as small as 1 byte
partObject = new S3Object(
testMultipartUpload.getObjectKey(), new byte[] {fiveMBTestData[0]});
uploadedPart = service.multipartUploadPart(
testMultipartUpload, ++partNumber, partObject);
assertEquals(uploadedPart.getPartNumber().longValue(), partNumber);
assertEquals(uploadedPart.getEtag(), partObject.getETag());
assertEquals(uploadedPart.getSize().longValue(), partObject.getContentLength());
// List multipart parts that have been received by the service
listedParts = service.multipartListParts(testMultipartUpload);
assertEquals(listedParts.size(), 3);
assertEquals(listedParts.get(2).getSize().longValue(), partObject.getContentLength());
// Reverse order of parts to ensure multipartCompleteUpload corrects the problem
Collections.reverse(listedParts);
// Complete multipart upload, despite badly ordered parts.
MultipartCompleted multipartCompleted = service.multipartCompleteUpload(
testMultipartUpload, listedParts);
assertEquals(multipartCompleted.getBucketName(), testMultipartUpload.getBucketName());
assertEquals(multipartCompleted.getObjectKey(), testMultipartUpload.getObjectKey());
// Confirm completed object exists and has expected size, metadata
S3Object completedObject = (S3Object) service.getObjectDetails(
bucketName, testMultipartUpload.getObjectKey());
assertEquals(completedObject.getContentLength(), fiveMBTestData.length * 2 + 1);
assertEquals(
metadata.get("test-md-value"),
completedObject.getMetadata("test-md-value"));
assertEquals(
metadata.get("test-timestamp-value").toString(),
completedObject.getMetadata("test-timestamp-value").toString());
// Confirm completed object has expected canned ACL settings
AccessControlList completedObjectACL =
service.getObjectAcl(bucketName, testMultipartUpload.getObjectKey());
assertTrue(completedObjectACL.hasGranteeAndPermission(
GroupGrantee.AUTHENTICATED_USERS, Permission.PERMISSION_READ));
} finally {
cleanupBucketForTest("testMultipartUploads");
}