Examples of org.jets3t.service.StorageObjectsChunk

org.jets3t.service.StorageObjectsChunk
Stores a "chunk" of StorageObjects returned from a list command - this particular chunk may or may not include all the objects available in a bucket. This class contains an array of objects and a the last key name returned by a prior call to the method {@link S3Service#listObjectsChunked(String,String,String,long,String)}. @author James Murty

      int maxListingLength, String priorLastKey) throws IOException {
    try {
      if (!prefix.isEmpty() && !prefix.endsWith(PATH_DELIMITER)) {
        prefix += PATH_DELIMITER;
      }
      StorageObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
          prefix, delimiter, maxListingLength, priorLastKey);
      
      FileMetadata[] fileMetadata =
        new FileMetadata[chunk.getObjects().length];
      for (int i = 0; i < fileMetadata.length; i++) {
        StorageObject object = chunk.getObjects()[i];
        fileMetadata[i] = new FileMetadata(object.getKey(),
            object.getContentLength(), object.getLastModifiedDate().getTime());
      }
      return new PartialListing(chunk.getPriorLastKey(), fileMetadata,
          chunk.getCommonPrefixes());
    } catch (ServiceException e) {
      handleException(e, prefix);
      return null; // never returned - keep compiler happy
    }
  }

View Full Code Here

      int maxListingLength, String priorLastKey) throws IOException {
    try {
      if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) {
        prefix += PATH_DELIMITER;
      }
      StorageObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
          prefix, delimiter, maxListingLength, priorLastKey);
      
      FileMetadata[] fileMetadata =
        new FileMetadata[chunk.getObjects().length];
      for (int i = 0; i < fileMetadata.length; i++) {
        StorageObject object = chunk.getObjects()[i];
        fileMetadata[i] = new FileMetadata(object.getKey(),
            object.getContentLength(), object.getLastModifiedDate().getTime());
      }
      return new PartialListing(chunk.getPriorLastKey(), fileMetadata,
          chunk.getCommonPrefixes());
    } catch (S3ServiceException e) {
      handleS3ServiceException(e);
      return null; //never returned - keep compiler happy
    } catch (ServiceException e) {
      handleServiceException(e);

View Full Code Here

        }
        if(automaticallyMergeChunks) {
            if(log.isDebugEnabled()) {
                log.debug("Found " + objects.size() + " objects in total");
            }
            return new StorageObjectsChunk(
                    prefix, delimiter,
                    objects.toArray(new StorageObject[objects.size()]),
                    commonPrefixes.toArray(new String[commonPrefixes.size()]),
                    null);
        }
        else {
            return new StorageObjectsChunk(
                    prefix, delimiter,
                    objects.toArray(new StorageObject[objects.size()]),
                    commonPrefixes.toArray(new String[commonPrefixes.size()]),
                    priorLastKey);
        }

View Full Code Here

                    final String delimiter = (String) filterObjectsDelimiter.getSelectedItem();


                    final ArrayList allObjects = new ArrayList();
                    String priorLastKey = null;
                    do {
                        StorageObjectsChunk chunk = s3ServiceMulti.getS3Service().listObjectsChunked(
                            currentSelectedBucket.getName(), prefix, delimiter,
                            BUCKET_LIST_CHUNKING_SIZE, priorLastKey);


                        final S3Object[] objects = S3Object.cast(chunk.getObjects());
                        for (int i = 0; i < objects.length; i++) {
                            objects[i].setOwner(currentSelectedBucket.getOwner());
                        }


                        priorLastKey = chunk.getPriorLastKey();
                        allObjects.addAll(Arrays.asList(objects));


                        updateProgressDialog(
                            "Listed " + allObjects.size() + " objects in "
                            + currentSelectedBucket.getName(), "", 0);

View Full Code Here

            // List items in chunks of size 2, ensure we get a total of seven.
            int chunkedObjectsCount = 0;
            int chunkedIterationsCount = 0;
            String priorLastKey = null;
            do {
                StorageObjectsChunk chunk = service.listObjectsChunked(
                    bucketName, null, null, 2, priorLastKey);
                priorLastKey = chunk.getPriorLastKey();
                chunkedObjectsCount += chunk.getObjects().length;
                chunkedIterationsCount++;
            } while (priorLastKey != null);
            assertEquals("Chunked bucket listing retreived incorrect number of objects",
                objectsList.size(), chunkedObjectsCount);
            assertEquals("Chunked bucket listing ran for an unexpected number of iterations",
                (objectsList.size() + 1) / 2, chunkedIterationsCount);


            // List objects with a prefix and delimiter to check common prefixes.
            StorageObjectsChunk chunk = service.listObjectsChunked(
                bucketName, "dir1/", "/", 100, null);
            assertEquals("Chunked bucket listing with prefix and delimiter retreived incorrect number of objects",
                3, chunk.getObjects().length);
            assertEquals("Chunked bucket listing with prefix and delimiter retreived incorrect number of common prefixes",
                1, chunk.getCommonPrefixes().length);


            // List the same items with a prefix.
            objects = service.listObjects(bucketName, "dir1", null);
            assertEquals("Incorrect number of objects matching prefix", 7, objects.length);

View Full Code Here

            try {
                List allObjects = new ArrayList();
                List allCommonPrefixes = new ArrayList();


                do {
                    StorageObjectsChunk chunk = storageService.listObjectsChunked(
                        bucketName, prefix, delimiter, maxListingLength, priorLastKey);
                    priorLastKey = chunk.getPriorLastKey();


                    allObjects.addAll(Arrays.asList(chunk.getObjects()));
                    allCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes()));
                } while (!halted && priorLastKey != null);


                result = new StorageObjectsChunk(
                    prefix, delimiter,
                    (StorageObject[]) allObjects.toArray(new StorageObject[allObjects.size()]),
                    (String[]) allCommonPrefixes.toArray(new String[allCommonPrefixes.size()]),
                    null);
            } catch (ServiceException e) {

View Full Code Here

         * standard object listing with a delimiter string.
         */
        long startTime = System.currentTimeMillis();


        // Find all the objects and common prefixes at the top level.
        StorageObjectsChunk initialChunk = restService.listObjectsChunked(
            bucketName, null, delimiter, 1000, null, true);


        long totalElapsedTime = System.currentTimeMillis() - startTime;


        // We will use the common prefix strings, if any, to perform sub-listings
        final String[] commonPrefixes = initialChunk.getCommonPrefixes();


        if (commonPrefixes.length > 0) {
            System.out.println("Performing sub-listings for common prefixes: "
                + Arrays.asList(commonPrefixes));


            /*
             * Create a S3ServiceMulti object with an event listener that responds to
             * ListObjectsEvent notifications and populates a complete object listing.
             */
            final S3ServiceMulti s3Multi = new S3ServiceMulti(restService, new S3ServiceEventAdaptor() {
                @Override
                public void s3ServiceEventPerformed(ListObjectsEvent event) {
                    if (ListObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {
                        Iterator chunkIter = event.getChunkList().iterator();
                        while (chunkIter.hasNext()) {
                            StorageObjectsChunk chunk = (StorageObjectsChunk) chunkIter.next();


                            System.out.println("Listed " + chunk.getObjects().length
                                + " objects for sub-listing with prefix: '"
                                + chunk.getPrefix() + "'");


                            allObjects.addAll(Arrays.asList(chunk.getObjects()));
                        }
                    } else if (ListObjectsEvent.EVENT_ERROR == event.getEventCode()) {
                        s3ServiceExceptions[0] = new S3ServiceException(
                            "Failed to list all objects in S3 bucket",
                            event.getErrorCause());

View Full Code Here

            @Override
            public void event(ListObjectsEvent event) {
                if (ListObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {
                    Iterator<StorageObjectsChunk> chunkIter = event.getChunkList().iterator();
                    while (chunkIter.hasNext()) {
                        StorageObjectsChunk chunk = chunkIter.next();


                        if (log.isDebugEnabled()) {
                            log.debug("Listed " + chunk.getObjects().length
                            + " objects and " + chunk.getCommonPrefixes().length
                            + " common prefixes in bucket '" + bucketName
                            + "' using prefix=" + chunk.getPrefix()
                            + ", delimiter=" + chunk.getDelimiter());
                        }


                        allObjects.addAll(Arrays.asList(chunk.getObjects()));
                        lastCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes()));
                    }
                } else if (ListObjectsEvent.EVENT_ERROR == event.getEventCode()) {
                    serviceExceptions[0] = new ServiceException(
                        "Failed to list all objects in bucket",
                        event.getErrorCause());

View Full Code Here

        StorageObject[] objects = null;
        String resultPriorLastKey = null;
        if (completeListing) {
            objects = listObjectsThreaded(service, bucketName, prefix);
        } else {
            StorageObjectsChunk chunk = service.listObjectsChunked(
                bucketName, prefix, null, Constants.DEFAULT_OBJECT_LIST_CHUNK_SIZE,
                priorLastKey, completeListing);
            objects = chunk.getObjects();
            resultPriorLastKey = chunk.getPriorLastKey();
        }


        Map<String, StorageObject> objectsMap = lookupObjectMetadataForPotentialClashes(
            service, bucketName, targetPath, objects, objectKeyToFilepathMap,
            forceMetadataDownload, isForceUpload, progressWatcher, eventListener);

View Full Code Here

            try {
                List allObjects = new ArrayList();
                List allCommonPrefixes = new ArrayList();


                do {
                    StorageObjectsChunk chunk = s3Service.listObjectsChunked(
                        bucketName, prefix, delimiter, maxListingLength, priorLastKey);
                    priorLastKey = chunk.getPriorLastKey();


                    allObjects.addAll(Arrays.asList(chunk.getObjects()));
                    allCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes()));
                } while (!halted && priorLastKey != null);


                result = new S3ObjectsChunk(
                    prefix, delimiter,
                    (S3Object[]) allObjects.toArray(new S3Object[allObjects.size()]),

View Full Code Here

0 1

TOP

Related Classes of org.jets3t.service.StorageObjectsChunk

org.apache.hadoop.fs.s3native.Jets3tNativeFileSystemStore

org.jets3t.apps.cockpit.Cockpit

org.jets3t.samples.ThreadedObjectListing

org.jets3t.service.BaseStorageServiceTests

org.jets3t.service.impl.rest.httpclient.RestStorageService

org.jets3t.service.multi.ThreadedStorageService$ListObjectsRunnable

org.jets3t.service.multithread.S3ServiceMulti$ListObjectsRunnable

org.jets3t.service.utils.FileComparer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.