Examples of MemoryChunk


Examples of com.salesforce.phoenix.memory.MemoryManager.MemoryChunk

    * @throws SQLException
    */
    SpoolingResultIterator(ResultIterator scanner, MemoryManager mm, final int thresholdBytes, final long maxSpoolToDisk) throws SQLException {
        boolean success = false;
        boolean usedOnDiskIterator = false;
        final MemoryChunk chunk = mm.allocate(0, thresholdBytes);
        File tempFile = null;
        try {
            // Can't be bigger than int, since it's the max of the above allocation
            int size = (int)chunk.getSize();
            tempFile = File.createTempFile("ResultSpooler",".bin");
            DeferredFileOutputStream spoolTo = new DeferredFileOutputStream(size, tempFile) {
                @Override
                protected void thresholdReached() throws IOException {
                    super.thresholdReached();
                    chunk.close();
                }
            };
            DataOutputStream out = new DataOutputStream(spoolTo);
            final long maxBytesAllowed = maxSpoolToDisk == -1 ?
                Long.MAX_VALUE : thresholdBytes + maxSpoolToDisk;
            long bytesWritten = 0L;
            int maxSize = 0;
            for (Tuple result = scanner.next(); result != null; result = scanner.next()) {
                int length = TupleUtil.write(result, out);
                bytesWritten += length;
                if(bytesWritten > maxBytesAllowed){
                    throw new SpoolTooBigToDiskException("result too big, max allowed(bytes): " + maxBytesAllowed);
                }
                maxSize = Math.max(length, maxSize);
            }
            spoolTo.close();
            if (spoolTo.isInMemory()) {
                byte[] data = spoolTo.getData();
                chunk.resize(data.length);
                spoolFrom = new InMemoryResultIterator(data, chunk);
            } else {
                spoolFrom = new OnDiskResultIterator(maxSize, spoolTo.getFile());
                usedOnDiskIterator = true;
            }
            success = true;
        } catch (IOException e) {
            throw ServerUtil.parseServerException(e);
        } finally {
            try {
                scanner.close();
            } finally {
                try {
                    if (!usedOnDiskIterator) {
                        tempFile.delete();
                    }
                } finally {
                    if (!success) {
                        chunk.close();
                    }
                }
            }
        }
    }
View Full Code Here

Examples of com.salesforce.phoenix.memory.MemoryManager.MemoryChunk

     */
    private RegionScanner getTopNScanner(final ObserverContext<RegionCoprocessorEnvironment> c, final RegionScanner s, final OrderedResultIterator iterator, ImmutableBytesWritable tenantId) throws Throwable {
        final Tuple firstTuple;
        TenantCache tenantCache = GlobalCache.getTenantCache(c.getEnvironment(), tenantId);
        long estSize = iterator.getEstimatedByteSize();
        final MemoryChunk chunk = tenantCache.getMemoryManager().allocate(estSize);
        final HRegion region = c.getEnvironment().getRegion();
        region.startRegionOperation();
        try {
            // Once we return from the first call to next, we've run through and cached
            // the topN rows, so we no longer need to start/stop a region operation.
            firstTuple = iterator.next();
            // Now that the topN are cached, we can resize based on the real size
            long actualSize = iterator.getByteSize();
            chunk.resize(actualSize);
        } catch (Throwable t) {
            ServerUtil.throwIOException(region.getRegionNameAsString(), t);
            return null;
        } finally {
            region.closeRegionOperation();
        }
        return new BaseRegionScanner() {
            private Tuple tuple = firstTuple;
           
            @Override
            public boolean isFilterDone() {
                return tuple == null;
            }

            @Override
            public HRegionInfo getRegionInfo() {
                return s.getRegionInfo();
            }

            @Override
            public boolean next(List<KeyValue> results) throws IOException {
                try {
                    if (isFilterDone()) {
                        return false;
                    }
                   
                    for (int i = 0; i < tuple.size(); i++) {
                        results.add(tuple.getValue(i));
                    }
                   
                    tuple = iterator.next();
                    return !isFilterDone();
                } catch (Throwable t) {
                    ServerUtil.throwIOException(region.getRegionNameAsString(), t);
                    return false;
                }
            }

            @Override
            public void close() throws IOException {
                try {
                    s.close();
                } finally {
                    chunk.close();                }
            }
        };
    }
View Full Code Here

Examples of com.salesforce.phoenix.memory.MemoryManager.MemoryChunk

        return getServerCaches().getIfPresent(cacheId);
    }
   
    @Override
    public Closeable addServerCache(ImmutableBytesPtr cacheId, ImmutableBytesWritable cachePtr, ServerCacheFactory cacheFactory) throws SQLException {
        MemoryChunk chunk = this.getMemoryManager().allocate(cachePtr.getLength());
        Closeable element = cacheFactory.newCache(cachePtr, chunk);
        getServerCaches().put(cacheId, element);
        return element;
    }
View Full Code Here

Examples of com.salesforce.phoenix.memory.MemoryManager.MemoryChunk

    }
   
    public ServerCache addServerCache(ScanRanges keyRanges, final ImmutableBytesWritable cachePtr, final ServerCacheFactory cacheFactory, final TableRef cacheUsingTableRef) throws SQLException {
        ConnectionQueryServices services = connection.getQueryServices();
        MemoryChunk chunk = services.getMemoryManager().allocate(cachePtr.getLength());
        List<Closeable> closeables = new ArrayList<Closeable>();
        closeables.add(chunk);
        ServerCache hashCacheSpec = null;
        SQLException firstException = null;
        final byte[] cacheId = generateId();
View Full Code Here

Examples of com.salesforce.phoenix.memory.MemoryManager.MemoryChunk

        } catch (InsufficientMemoryException e) { // expected
        }

        ChildMemoryManager rmm1 = new ChildMemoryManager(gmm,100);
        ChildMemoryManager rmm2 = new ChildMemoryManager(gmm,100);
        MemoryChunk c1 = rmm1.allocate(100);
        MemoryChunk c2 = rmm2.allocate(100);
        try {
            rmm2.allocate(100);
            fail();
        } catch (InsufficientMemoryException e) { // expected
        }
       
        c1.close();
        c2.close();
        assertTrue(rmm1.getAvailableMemory() == rmm1.getMaxMemory());
    }
View Full Code Here

Examples of com.salesforce.phoenix.memory.MemoryManager.MemoryChunk

        final ChildMemoryManager rmm1 = new ChildMemoryManager(gmm,100);
        final ChildMemoryManager rmm2 = new ChildMemoryManager(gmm,100);
        Thread t1 = new Thread() {
            @Override
            public void run() {
                MemoryChunk c1 = rmm1.allocate(50);
                MemoryChunk c2 = rmm1.allocate(50);
                sleepFor(4000);
                c1.close();
                sleepFor(2000);
                c2.close();
            }
        };
        Thread t2 = new Thread() {
            @Override
            public void run() {
                sleepFor(2000);
                // Will require waiting for a bit of time before t1 frees the requested memory
                long startTime = System.currentTimeMillis();
                MemoryChunk c3 = rmm2.allocate(50);
                assertTrue(System.currentTimeMillis() - startTime >= 1000);
                c3.close();
            }
        };
        t1.start();
        t2.start();
        sleepFor(1000);
        // Main thread competes with others to get all memory, but should wait
        // until both threads are complete (since that's when the memory will
        // again be all available.
        ChildMemoryManager rmm = new ChildMemoryManager(gmm,100);
        MemoryChunk c = rmm.allocate(100);
        c.close();
        assertTrue(rmm.getAvailableMemory() == rmm.getMaxMemory());
        assertTrue(rmm1.getAvailableMemory() == rmm1.getMaxMemory());
        assertTrue(rmm2.getAvailableMemory() == rmm2.getMaxMemory());
    }
View Full Code Here

Examples of com.salesforce.phoenix.memory.MemoryManager.MemoryChunk

        final ChildMemoryManager rmm1 = new ChildMemoryManager(gmm,100);
        final ChildMemoryManager rmm2 = new ChildMemoryManager(gmm,100);
        Thread t1 = new Thread() {
            @Override
            public void run() {
                MemoryChunk c1 = rmm1.allocate(50);
                MemoryChunk c2 = rmm1.allocate(40);
                sleepFor(4000);
                c1.close();
                sleepFor(2000);
                c2.close();
            }
        };
        Thread t2 = new Thread() {
            @Override
            public void run() {
                sleepFor(2000);
                MemoryChunk c3 = rmm2.allocate(10);
                // Will require waiting for a bit of time before t1 frees the requested memory
                long startTime = System.currentTimeMillis();
                c3.resize(50);
                assertTrue(System.currentTimeMillis() - startTime >= 2000);
                c3.close();
            }
        };
        t1.start();
        t2.start();
        sleepFor(3000);
        // Main thread competes with others to get all memory, but should wait
        // until both threads are complete (since that's when the memory will
        // again be all available.
        ChildMemoryManager rmm = new ChildMemoryManager(gmm,100);
        MemoryChunk c = rmm.allocate(100);
        c.close();
        assertTrue(rmm.getAvailableMemory() == rmm.getMaxMemory());
        assertTrue(rmm1.getAvailableMemory() == rmm1.getMaxMemory());
        assertTrue(rmm2.getAvailableMemory() == rmm2.getMaxMemory());
    }
View Full Code Here

Examples of com.salesforce.phoenix.memory.MemoryManager.MemoryChunk

   
    // @Test commenting out because the test is flapping too often
    public void broken_testWaitUntilResize() {
        final GlobalMemoryManager gmm = new GlobalMemoryManager(100,8000);
        final ChildMemoryManager rmm1 = new ChildMemoryManager(gmm,100);
        final MemoryChunk c1 = rmm1.allocate(70);
        Thread t1 = new Thread() {
            @Override
            public void run() {
                MemoryChunk c2 = rmm1.allocate(20);
                sleepFor(4000);
                c1.resize(20); // resize down to test that other thread is notified
                sleepFor(2000);
                c2.close();
                c1.close();
                assertTrue(rmm1.getAvailableMemory() == rmm1.getMaxMemory());
            }
        };
        Thread t2 = new Thread() {
            @Override
            public void run() {
                sleepFor(2000);
                ChildMemoryManager rmm2 = new ChildMemoryManager(gmm,100);
                MemoryChunk c3 = rmm2.allocate(10);
                long startTime = System.currentTimeMillis();
                c3.resize(60); // Test that resize waits if memory not available
                assertTrue(c1.getSize() == 20); // c1 was resized not closed
                assertTrue(System.currentTimeMillis() - startTime >= 2000); // we waited some time before the allocate happened
                c3.close();
                assertTrue(rmm2.getAvailableMemory() == rmm2.getMaxMemory());
            }
        };
        t1.start();
        t2.start();
        sleepFor(1000);
        // Main thread competes with others to get all memory, but should wait
        // until both threads are complete (since that's when the memory will
        // again be all available.
        ChildMemoryManager rmm = new ChildMemoryManager(gmm,100);
        MemoryChunk c = rmm.allocate(100);
        c.close();
        assertTrue(rmm.getAvailableMemory() == rmm.getMaxMemory());
    }
View Full Code Here

Examples of com.salesforce.phoenix.memory.MemoryManager.MemoryChunk

    @Test
    public void testChildDecreaseAllocation() throws Exception {
        MemoryManager gmm = new GlobalMemoryManager(100,1);
        ChildMemoryManager rmm1 = new ChildMemoryManager(gmm,100);
        ChildMemoryManager rmm2 = new ChildMemoryManager(gmm,10);
        MemoryChunk c1 = rmm1.allocate(50);
        MemoryChunk c2 = rmm2.allocate(5,50);
        assertTrue(c2.getSize() == 10);
        c1.close();
        assertTrue(rmm1.getAvailableMemory() == rmm1.getMaxMemory());
        c2.close();
        assertTrue(rmm2.getAvailableMemory() == rmm2.getMaxMemory());
        assertTrue(gmm.getAvailableMemory() == gmm.getMaxMemory());
    }
View Full Code Here

Examples of com.salesforce.phoenix.memory.MemoryManager.MemoryChunk

        MemoryManager gmm = new GlobalMemoryManager(100,1);
        ChildMemoryManager rmm1 = new ChildMemoryManager(gmm,25);
        ChildMemoryManager rmm2 = new ChildMemoryManager(gmm,25);
        ChildMemoryManager rmm3 = new ChildMemoryManager(gmm,25);
        ChildMemoryManager rmm4 = new ChildMemoryManager(gmm,35);
        MemoryChunk c1 = rmm1.allocate(20);
        MemoryChunk c2 = rmm2.allocate(20);
        try {
            rmm1.allocate(10);
            fail();
        } catch (InsufficientMemoryException e) { // expected
        }
        MemoryChunk c3 = rmm3.allocate(25);
        c1.close();
        // Ensure that you can get back to max for rmn1 after failure
        MemoryChunk c4 = rmm1.allocate(10);
        MemoryChunk c5 = rmm1.allocate(15);
       
        MemoryChunk c6 = rmm4.allocate(25);
        try {
            // This passes % test, but fails the next total memory usage test
            rmm4.allocate(10);
            fail();
        } catch (InsufficientMemoryException e) { // expected
        }
        c2.close();
        // Tests that % test passes (confirming that the 10 above was subtracted back from request memory usage,
        // since we'd be at the max of 35% now
        MemoryChunk c7 = rmm4.allocate(10);
       
        try {
            rmm4.allocate(1);
            fail();
        } catch (InsufficientMemoryException e) { // expected
        }

        try {
            rmm2.allocate(25);
            fail();
        } catch (InsufficientMemoryException e) { // expected
        }

        c3.close();
        c4.close();
        c5.close();
        c6.close();
        c7.close();
        assertTrue(rmm1.getAvailableMemory() == rmm1.getMaxMemory());
        assertTrue(rmm2.getAvailableMemory() == rmm2.getMaxMemory());
        assertTrue(rmm3.getAvailableMemory() == rmm3.getMaxMemory());
        assertTrue(rmm4.getAvailableMemory() == rmm4.getMaxMemory());
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.