Examples of AmazonS3

com.amazonaws.services.s3.AmazonS3
mazon.com/s3"> http://aws.amazon.com/s3
org.eclipse.jgit.transport.AmazonS3
t.s3.amazonaws.com/index.html">jets3t, a popular Java based Amazon S3 client library. Enabling encryption can hide sensitive data from the operators of the S3 service.

Examples of com.amazonaws.services.s3.AmazonS3

     * 
     * @param bucketName the bucket name.
     */
    public static void deleteBucket(final String bucketName) throws IOException {
        Properties prop = readConfig(DEFAULT_CONFIG_FILE);
        AmazonS3 s3service = openService(prop);
        ObjectListing prevObjectListing = s3service.listObjects(bucketName);
        while (true) {
            for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                s3service.deleteObject(bucketName, s3ObjSumm.getKey());
            }
            if (!prevObjectListing.isTruncated()) {
                break;
            }
            prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
        }
        s3service.deleteBucket(bucketName);
    }

View Full Code Here

Examples of com.amazonaws.services.s3.AmazonS3

    {
        try
        {
            logger.info("Downloading " + path.getRemotePath());
            downloadCount.incrementAndGet();
            final AmazonS3 client = getS3Client();
            long contentLen = client.getObjectMetadata(getPrefix(), path.getRemotePath()).getContentLength();
            path.setSize(contentLen);
            RangeReadInputStream rris = new RangeReadInputStream(client, getPrefix(), path);
            final long bufSize = MAX_BUFFERED_IN_STREAM_SIZE > contentLen ? contentLen : MAX_BUFFERED_IN_STREAM_SIZE;
            compress.decompressAndClose(new BufferedInputStream(rris, (int)bufSize), os);
            bytesDownloaded.addAndGet(contentLen);

View Full Code Here

Examples of com.amazonaws.services.s3.AmazonS3


    @Override
    public void upload(AbstractBackupPath path, InputStream in) throws BackupRestoreException
    {
        uploadCount.incrementAndGet();
        AmazonS3 s3Client = getS3Client();
        InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(config.getBackupPrefix(), path.getRemotePath());
        InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);
        DataPart part = new DataPart(config.getBackupPrefix(), path.getRemotePath(), initResponse.getUploadId());
        List<PartETag> partETags = Lists.newArrayList();
        long chunkSize = config.getBackupChunkSize();
        if (path.getSize() > 0)
            chunkSize = (path.getSize() / chunkSize >= MAX_CHUNKS) ? (path.getSize() / (MAX_CHUNKS - 1)) : chunkSize;
        logger.info(String.format("Uploading to %s/%s with chunk size %d", config.getBackupPrefix(), path.getRemotePath(), chunkSize));
        try
        {
            Iterator<byte[]> chunks = compress.compress(in, chunkSize);
            // Upload parts.
            int partNum = 0;
            while (chunks.hasNext())
            {
                byte[] chunk = chunks.next();
                rateLimiter.acquire(chunk.length);
                DataPart dp = new DataPart(++partNum, chunk, config.getBackupPrefix(), path.getRemotePath(), initResponse.getUploadId());
                S3PartUploader partUploader = new S3PartUploader(s3Client, dp, partETags);
                executor.submit(partUploader);
                bytesUploaded.addAndGet(chunk.length);
            }
            executor.sleepTillEmpty();
            if (partNum != partETags.size())
                throw new BackupRestoreException("Number of parts(" + partNum + ")  does not match the uploaded parts(" + partETags.size() + ")");
            new S3PartUploader(s3Client, part, partETags).completeUpload();
            
            if (logger.isDebugEnabled())
            {  
               final S3ResponseMetadata responseMetadata = s3Client.getCachedResponseMetadata(initRequest);
               final String requestId = responseMetadata.getRequestId(); // "x-amz-request-id" header
               final String hostId = responseMetadata.getHostId(); // "x-amz-id-2" header
               logger.debug("S3 AWS x-amz-request-id[" + requestId + "], and x-amz-id-2[" + hostId + "]");
            }

View Full Code Here

Examples of com.amazonaws.services.s3.AmazonS3

     * set. Removes the rule is set to 0.
     */
    @Override
    public void cleanup()
    {
        AmazonS3 s3Client = getS3Client();
        String clusterPath = pathProvider.get().clusterPrefix("");
        BucketLifecycleConfiguration lifeConfig = s3Client.getBucketLifecycleConfiguration(config.getBackupPrefix());
        if (lifeConfig == null)
        {
            lifeConfig = new BucketLifecycleConfiguration();
            List<Rule> rules = Lists.newArrayList();
            lifeConfig.setRules(rules);
        }
        List<Rule> rules = lifeConfig.getRules();
        if (updateLifecycleRule(rules, clusterPath))
        {
            if( rules.size() > 0 ){
                lifeConfig.setRules(rules);
                s3Client.setBucketLifecycleConfiguration(config.getBackupPrefix(), lifeConfig);
            }
            else
                s3Client.deleteBucketLifecycleConfiguration(config.getBackupPrefix());
        }
    }

View Full Code Here

Examples of com.amazonaws.services.s3.AmazonS3

        String bucket = req.getParameter("bucket");


        resp.setStatus(200);


        AWSCredentials myCredentials = new BasicAWSCredentials(AWS_PUBLIC_KEY_SERVER, AWS_SECRET_KEY_SERVER);
        AmazonS3 s3Client = new AmazonS3Client(myCredentials);
        s3Client.deleteObject(bucket, key);
    }

View Full Code Here

Examples of com.amazonaws.services.s3.AmazonS3

        System.out.println(String.format("Upload successfully sent to S3!  Bucket: %s, Key: %s, UUID: %s, Filename: %s",
                bucket, key, uuid, name));


        AWSCredentials myCredentials = new BasicAWSCredentials(
                                        AWS_PUBLIC_KEY_SERVER, AWS_SECRET_KEY_SERVER);
        AmazonS3 s3Client = new AmazonS3Client(myCredentials);


        java.util.Date expiration = new java.util.Date();
        long msec = expiration.getTime();
        msec += 1000 * 60 * 60; // 1 hour.
        expiration.setTime(msec);


        GeneratePresignedUrlRequest generatePresignedUrlRequest =
                      new GeneratePresignedUrlRequest(bucket, key);
        generatePresignedUrlRequest.setMethod(HttpMethod.GET); // Default.
        generatePresignedUrlRequest.setExpiration(expiration);


        URL s = s3Client.generatePresignedUrl(generatePresignedUrlRequest);


        JsonObject response = new JsonObject();
        response.addProperty("thumbnailUrl", s.toString());


        resp.getWriter().write(response.toString());

View Full Code Here

Examples of com.amazonaws.services.s3.AmazonS3

                            em.persist(_item);
                        }
                    }
                }
            } else if (field.isJsonLob()) {
                AmazonS3 s3 = em.getS3Service();


                long start3 = System.currentTimeMillis();
                String bucketName = em.getS3BucketName();
                String classRef = ai.getRootClass().getName().replace('.', '/');
                long lobTimestamp = System.currentTimeMillis() / 1000;
                String s3ObjectId = String.format("%s/%s/%08X.json", classRef, id + "-" + field.getFieldName(),
                        lobTimestamp);


                byte[] contentBytes = getJsonPayload(ob);


                InputStream input = new ByteArrayInputStream(contentBytes);


                s3.putObject(bucketName, s3ObjectId, input, null);


                em.statsS3Put(System.currentTimeMillis() - start3);
                logger.finer("setting lobkeyattribute=" + columnName + " - " + s3ObjectId);
                attsToPut.add(new ReplaceableAttribute(columnName, s3ObjectId, true));
            } else if (field.isLob()) {
                // store in s3
                AmazonS3 s3 = null;
                // todo: need to make sure we only store to S3 if it's changed,
                // too slow.
                logger.fine("putting lob to s3");
                long start3 = System.currentTimeMillis();
                s3 = em.getS3Service();
                String bucketName = em.getS3BucketName();
                String s3ObjectId = id + "-" + field.getFieldName();


                ByteArrayOutputStream bos = new ByteArrayOutputStream();
                ObjectOutputStream out = new ObjectOutputStream(bos);
                out.writeObject(ob);
                byte[] contentBytes = bos.toByteArray();
                out.close();
                InputStream input = new ByteArrayInputStream(contentBytes);


                s3.putObject(bucketName, s3ObjectId, input, null);


                em.statsS3Put(System.currentTimeMillis() - start3);
                logger.finer("setting lobkeyattribute=" + columnName + " - " + s3ObjectId);
                attsToPut.add(new ReplaceableAttribute(columnName, s3ObjectId, true));
            } else if (field.getEnumType() != null) {

View Full Code Here

Examples of com.amazonaws.services.s3.AmazonS3

        return factory.getExecutor();
    }


    public Object getObjectFromS3(String idOnS3) throws AmazonClientException, IOException, ClassNotFoundException {
        long start = System.currentTimeMillis();
        AmazonS3 s3 = factory.getS3Service();
        S3Object s3o = s3.getObject(factory.getS3BucketName(), idOnS3);
        logger.fine("got s3object=" + s3o);
        Object ret = null;
        try {
            ObjectInputStream reader = new ObjectInputStream(new BufferedInputStream((s3o.getObjectContent())));
            ret = reader.readObject();

View Full Code Here

Examples of org.eclipse.jgit.transport.AmazonS3

    return false;
  }


  @Override
  protected void run() throws Exception {
    final AmazonS3 s3 = new AmazonS3(properties());


    if ("get".equals(op)) { //$NON-NLS-1$
      final URLConnection c = s3.get(bucket, key);
      int len = c.getContentLength();
      final InputStream in = c.getInputStream();
      try {
        outw.flush();
        final byte[] tmp = new byte[2048];
        while (len > 0) {
          final int n = in.read(tmp);
          if (n < 0)
            throw new EOFException(MessageFormat.format(
                CLIText.get().expectedNumberOfbytes,
                valueOf(len)));
          outs.write(tmp, 0, n);
          len -= n;
        }
        outs.flush();
      } finally {
        in.close();
      }


    } else if ("ls".equals(op) || "list".equals(op)) { //$NON-NLS-1$//$NON-NLS-2$
      for (final String k : s3.list(bucket, key))
        outw.println(k);


    } else if ("rm".equals(op) || "delete".equals(op)) { //$NON-NLS-1$ //$NON-NLS-2$
      s3.delete(bucket, key);


    } else if ("put".equals(op)) { //$NON-NLS-1$
      final OutputStream os = s3.beginPut(bucket, key, null, null);
      final byte[] tmp = new byte[2048];
      int n;
      while ((n = System.in.read(tmp)) > 0)
        os.write(tmp, 0, n);
      os.close();

View Full Code Here

Examples of org.eclipse.jgit.transport.AmazonS3

    return false;
  }


  @Override
  protected void run() throws Exception {
    final AmazonS3 s3 = new AmazonS3(properties());


    if ("get".equals(op)) { //$NON-NLS-1$
      final URLConnection c = s3.get(bucket, key);
      int len = c.getContentLength();
      final InputStream in = c.getInputStream();
      try {
        outw.flush();
        final byte[] tmp = new byte[2048];
        while (len > 0) {
          final int n = in.read(tmp);
          if (n < 0)
            throw new EOFException(MessageFormat.format(
                CLIText.get().expectedNumberOfbytes,
                valueOf(len)));
          outs.write(tmp, 0, n);
          len -= n;
        }
        outs.flush();
      } finally {
        in.close();
      }


    } else if ("ls".equals(op) || "list".equals(op)) { //$NON-NLS-1$//$NON-NLS-2$
      for (final String k : s3.list(bucket, key))
        outw.println(k);


    } else if ("rm".equals(op) || "delete".equals(op)) { //$NON-NLS-1$ //$NON-NLS-2$
      s3.delete(bucket, key);


    } else if ("put".equals(op)) { //$NON-NLS-1$
      final OutputStream os = s3.beginPut(bucket, key, null, null);
      final byte[] tmp = new byte[2048];
      int n;
      while ((n = ins.read(tmp)) > 0)
        os.write(tmp, 0, n);
      os.close();

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.