Package com.amazonaws.services.s3.model.DeleteObjectsRequest

Examples of com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion


     *
     * @return AmazonS3Client
     */
    AmazonS3 createS3Client() {
        AWSCredentials credentials = new BasicAWSCredentials(configuration.getAccessKey(), configuration.getSecretKey());
        AmazonS3 client = new AmazonS3Client(credentials);
        return client;
    }
View Full Code Here


  assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));

  Settings settings = internalCluster().getInstance(Settings.class);
  Settings bucket = settings.getByPrefix("repositories.s3.");
  AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(
    bucket.get("region", settings.get("repositories.s3.region")),
    bucket.get("access_key", settings.get("cloud.aws.access_key")),
    bucket.get("secret_key", settings.get("cloud.aws.secret_key")));

    String bucketName = bucket.get("bucket");
  logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath);
  List<S3ObjectSummary> summaries = s3Client.listObjects(bucketName, basePath).getObjectSummaries();
  for (S3ObjectSummary summary : summaries) {
      assertThat(s3Client.getObjectMetadata(bucketName, summary.getKey()).getSSEAlgorithm(), equalTo("AES256"));
  }

  logger.info("--> delete some data");
  for (int i = 0; i < 50; i++) {
      client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get();
View Full Code Here

            String bucketName = bucket.get("bucket");

            // We check that settings has been set in elasticsearch.yml integration test file
            // as described in README
            assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue());
            AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(region, accessKey, secretKey);
            try {
                ObjectListing prevListing = null;
                //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
                //we can do at most 1K objects per delete
                //We don't know the bucket name until first object listing
                DeleteObjectsRequest multiObjectDeleteRequest = null;
                ArrayList<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<DeleteObjectsRequest.KeyVersion>();
                while (true) {
                    ObjectListing list;
                    if (prevListing != null) {
                        list = client.listNextBatchOfObjects(prevListing);
                    } else {
                        list = client.listObjects(bucketName, basePath);
                        multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
                    }
                    for (S3ObjectSummary summary : list.getObjectSummaries()) {
                        keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
                        //Every 500 objects batch the delete request
                        if (keys.size() > 500) {
                            multiObjectDeleteRequest.setKeys(keys);
                            client.deleteObjects(multiObjectDeleteRequest);
                            multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
                            keys.clear();
                        }
                    }
                    if (list.isTruncated()) {
                        prevListing = list;
                    } else {
                        break;
                    }
                }
                if (!keys.isEmpty()) {
                    multiObjectDeleteRequest.setKeys(keys);
                    client.deleteObjects(multiObjectDeleteRequest);
                }
            } catch (Throwable ex) {
                logger.warn("Failed to delete S3 repository [{}] in [{}]", ex, bucketName, region);
            }
        }
View Full Code Here

     *
     * @return AmazonS3Client
     */
    AmazonS3 createS3Client() {
        AWSCredentials credentials = new BasicAWSCredentials(configuration.getAccessKey(), configuration.getSecretKey());
        AmazonS3 client = new AmazonS3Client(credentials);
        return client;
    }
View Full Code Here

    }


    private synchronized AmazonS3 getClient(String endpoint, String account, String key) {
        Tuple<String, String> clientDescriptor = new Tuple<String, String>(endpoint, account);
        AmazonS3Client client = clients.get(clientDescriptor);
        if (client != null) {
            return client;
        }

        ClientConfiguration clientConfiguration = new ClientConfiguration();
        String protocol = componentSettings.get("protocol", "https").toLowerCase();
        protocol = componentSettings.get("s3.protocol", protocol).toLowerCase();
        if ("http".equals(protocol)) {
            clientConfiguration.setProtocol(Protocol.HTTP);
        } else if ("https".equals(protocol)) {
            clientConfiguration.setProtocol(Protocol.HTTPS);
        } else {
            throw new ElasticsearchIllegalArgumentException("No protocol supported [" + protocol + "], can either be [http] or [https]");
        }

        String proxyHost = componentSettings.get("proxy_host");
        if (proxyHost != null) {
            String portString = componentSettings.get("proxy_port", "80");
            Integer proxyPort;
            try {
                proxyPort = Integer.parseInt(portString, 10);
            } catch (NumberFormatException ex) {
                throw new ElasticsearchIllegalArgumentException("The configured proxy port value [" + portString + "] is invalid", ex);
            }
            clientConfiguration.withProxyHost(proxyHost).setProxyPort(proxyPort);
        }

        AWSCredentialsProvider credentials;

        if (account == null && key == null) {
            credentials = new AWSCredentialsProviderChain(
                    new EnvironmentVariableCredentialsProvider(),
                    new SystemPropertiesCredentialsProvider(),
                    new InstanceProfileCredentialsProvider()
            );
        } else {
            credentials = new AWSCredentialsProviderChain(
                    new StaticCredentialsProvider(new BasicAWSCredentials(account, key))
            );
        }
        client = new AmazonS3Client(credentials, clientConfiguration);

        if (endpoint != null) {
            client.setEndpoint(endpoint);
        }
        clients.put(clientDescriptor, client);
        return client;
    }
View Full Code Here

     *
     * @return AmazonS3Client
     */
    AmazonS3Client createS3Client() {
        AWSCredentials credentials = new BasicAWSCredentials(configuration.getAccessKey(), configuration.getSecretKey());
        AmazonS3Client client = new AmazonS3Client(credentials);
        if (configuration.getAmazonS3Endpoint() != null) {
            client.setEndpoint(configuration.getAmazonS3Endpoint());
        }
        return client;
    }
View Full Code Here

        CopyObjectResultHandler copyObjectResultHandler = null;
        try {
            // TODO: Should we move some more of this logic into CopyObjectResponseHandler?
            //       For example, detecting the different failure modes?
            CopyObjectResponseHandler responseHandler = new CopyObjectResponseHandler();
            copyObjectResultHandler =
                (CopyObjectResultHandler)client.execute(httpRequest, responseHandler, errorResponseHandler);
        } catch (AmazonS3Exception ase) {
            /*
             * If the request failed because one of the specified constraints
View Full Code Here

             * buffer the whole stream in memory just to calculate it. Instead,
             * we can calculate it on the fly and validate it with the returned
             * ETag from the object upload.
             */
            try {
                input = new MD5DigestCalculatingInputStream(input);
            } catch (NoSuchAlgorithmException e) {
                log.warn("No MD5 digest algorithm available.  Unable to calculate " +
                         "checksum and verify data integrity.", e);
            }
        }

        if (metadata.getContentType() == null) {
            /*
             * Default to the "application/octet-stream" if the user hasn't
             * specified a content type.
             */
            metadata.setContentType(Mimetypes.MIMETYPE_OCTET_STREAM);
        }

        populateRequestMetadata(request, metadata);
        signRequest(request, HttpMethodName.PUT, bucketName, key);
        HttpRequest httpRequest = convertToHttpRequest(request, HttpMethodName.PUT);
        httpRequest.setContent(input);

        ObjectMetadata returnedMetadata = null;
        try {
            S3MetadataResponseHandler responseHandler = new S3MetadataResponseHandler();
            returnedMetadata = (ObjectMetadata)client.execute(httpRequest, responseHandler, errorResponseHandler);
        } finally {
            try {input.close();} catch (Exception e) {
                log.warn("Unable to cleanly close input stream: " + e.getMessage(), e);
            }
        }

        String contentMd5 = metadata.getContentMD5();
        if (input instanceof MD5DigestCalculatingInputStream) {
            MD5DigestCalculatingInputStream md5DigestInputStream = (MD5DigestCalculatingInputStream)input;
            contentMd5 = ServiceUtils.toBase64(md5DigestInputStream.getMd5Digest());
        }

        if (returnedMetadata != null && contentMd5 != null) {
            byte[] clientSideHash = ServiceUtils.fromBase64(contentMd5);
            byte[] serverSideHash = ServiceUtils.fromHex(returnedMetadata.getETag());
View Full Code Here

                throw new AmazonClientException(
                        "Unable to calculate MD5 hash: " + e.getMessage(), e);
            }

            try {
                input = new RepeatableFileInputStream(file);
            } catch (FileNotFoundException fnfe) {
                throw new AmazonClientException("Unable to find file to upload", fnfe);
            }
        }
View Full Code Here

                     "Stream contents will be buffered in memory and could result in " +
                     "out of memory errors.");
        }

        if (!input.markSupported()) {
            input = new RepeatableInputStream(input, Constants.DEFAULT_STREAM_BUFFER_SIZE);
        }

        if (metadata.getContentMD5() == null) {
            /*
             * If the user hasn't set the content MD5, then we don't want to
View Full Code Here

TOP

Related Classes of com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.