package com.cloudhopper.mq.queue.impl;
/*
* #%L
* ch-mq
* %%
* Copyright (C) 2012 Cloudhopper by Twitter
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import com.cloudhopper.mq.message.PriorityMQMessage;
import com.cloudhopper.mq.queue.*;
import com.cloudhopper.mq.util.CompositeKey;
import com.cloudhopper.mq.util.CompositeKeyUtil;
import com.cloudhopper.mq.util.PriorityCompositeKeyUtil;
import com.cloudhopper.datastore.DataStore;
import com.cloudhopper.datastore.DataStoreIterator;
import com.cloudhopper.datastore.DataStoreFatalException;
import com.cloudhopper.datastore.RecordNotFoundException;
import com.google.common.collect.MinMaxPriorityQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* Support for getting rid of objects in memory from the "queue" if they aren't
* being read fast enough. Persist them to disk and don't put them in the queue.
* Perhaps start some sort of thread behind the scenes to load them back in as
* needed. In the DefaultPriorityQueue implementation, this could be done with the
* following simple mechanism, as we have a sorted key that doesn't change:
* - given a "page" with a capacity of N elements, "overflow" boolean starts = false
* - Keep a "head" and "tail" value
* - Keep a "size" counter.
* - On PUT
* - if the page is full (size = max capacity)
* - set "overflow" = true
* - if "tail".compareTo("put item") > 0
* - write "put item" to DataStore
* - if "tail".compareTo("put item") < 0
* - write "put item" to page and DataStore
* - removed "tail" from page
* - if the page is not full
* - write to page and DataStore
* - On TAKE
* - if the page TAKE succeeds, delete from page and DataStore
* - if the page TAKE fails AND "overflow"==true, load a page from DataStore
* (by using DataStoreIterator, constrained by pagesize)
* - if the page load from DataStore does not load all values, set "overflow"
* boolean value = false
*
* @author garth
*/
public class BoundedMemoryPriorityQueue<E> extends DefaultPriorityQueue<E> {
private static final Logger logger = LoggerFactory.getLogger(BoundedMemoryPriorityQueue.class);
// in-memory priority queue
private MinMaxPriorityQueue<PriorityMQMessage> queue = MinMaxPriorityQueue.<PriorityMQMessage>create();
// paging
private boolean overflow = false;
private int maxItemsInMemory = 10000;
private AtomicLong memorySize = new AtomicLong(0L);
private AtomicLong overflowSize = new AtomicLong(0L);
private AtomicLong pageLoadedCount = new AtomicLong(0L);
public BoundedMemoryPriorityQueue() {
super();
}
/**
* Gets the number of times that this queue has loaded a page from persistent, overflow storage.
*/
public long getPagesLoaded() {
return pageLoadedCount.get();
}
/**
* Get the number of items currenty in memory.
*/
public long getMemorySize() {
return memorySize.get();
}
/**
* Get the number of items currenty in persistent, overflow storage. This does not include the number
* of stored items that are also in memory. Thus, getMemorySize() + getOverflowSize() = getSize().
*/
public long getOverflowSize() {
return overflowSize.get();
}
public static final String PROP_MAX_ITEMS_IN_MEMORY = "maxItemsInMemory";
@Override
public void setProperty(String name, Object value) {
if (PROP_MAX_ITEMS_IN_MEMORY.equals(name)) {
maxItemsInMemory = Integer.parseInt(value.toString());
}
}
@Override
protected void doActivate() throws Exception {
this.size.set(memorySize.get() + overflowSize.get());
this.putCount.set(memorySize.get() + overflowSize.get());
}
@Override
public void preload(DataStoreIterator iterator) throws DataStoreFatalException, QueueFatalException {
int itemsLoaded = 0;
while (iterator.next()) {
DataStoreIterator.Record record = iterator.getRecord();
CompositeKey key = priorityKeyUtil.decode(record.getKey());
PriorityMQMessage element = null;
try {
element = priorityTranscoder.decode(record.getValue());
} catch (Throwable t) {
throw new QueueFatalException("Unable to decode element with transcoder for queueId " + getId() + ". Perhaps incorrect transcoder?", t);
}
// assume the iterator is moving in ascending key order (also priority order) and just cut off after maxItemsInMemory is loaded
if (itemsLoaded <= maxItemsInMemory) {
nextSequence(element);
queue.add(element);
memorySize.incrementAndGet();
} else {
overflowSize.incrementAndGet();
}
}
}
@Override
protected boolean doStore(PriorityMQMessage<E> item, byte[] encoded) throws QueueIsFullException {
long itemId = item.key();
// - On PUT
// - if the page is full (size = max capacity)
// - set "overflow" = true
// - if "tail".compareTo("put item") > 0
// - write "put item" to DataStore
// - if "tail".compareTo("put item") < 0
// - write "put item" to page and DataStore
// - removed "tail" from page
// - if the page is not full
// - write to page and DataStore
boolean dsOnly = false;
boolean replaceLast = false;
int qsize = queue.size(); //maybe use memoryCount.get() instead of queue.size()? which is more performant?
if (qsize != 0 && qsize == maxItemsInMemory) {
if (!overflow) logger.trace("Overflow at {}", qsize);
overflow = true;
PriorityMQMessage tail = queue.peekLast();
int compare = item.compareTo(tail);
if (compare >= 0) {
dsOnly = true;
} else if (compare < 0) {
replaceLast = true;
logger.trace("Replacing tail ({})", compare);
}
}
byte[] key = priorityKeyUtil.encode(getId(), itemId);
try {
ds.setRecord(key, encoded);
} catch (DataStoreFatalException e) {
logger.error("Unable to permanently store key and value for queueId=" + getId() + ", itemId=" + itemId, e);
this.errorCount.incrementAndGet();
}
if (overflow) overflowSize.incrementAndGet();
// add the item to the in-memory queue
if (replaceLast) {
queue.removeLast();
memorySize.decrementAndGet(); //we might turn around and increment in, but we need to decrement it for now
}
if (!dsOnly) {
queue.add(item);
memorySize.incrementAndGet();
}
return true;
}
@Override
protected PriorityMQMessage<E> doTake() {
PriorityMQMessage w = queue.peek();
long itemId = w.key();
logger.trace("[{}] itemId={} in queue.take()", getName(), itemId);
byte[] key = priorityKeyUtil.encode(getId(), itemId);
try {
ds.deleteRecord(key);
} catch (RecordNotFoundException e) {
logger.error("Key not found in DataStore for queueId=" + getId() + ", itemId=" + itemId, e);
this.errorCount.incrementAndGet();
} catch (DataStoreFatalException e) {
logger.error("Unable to permanently delete key for queueId=" + getId() + ", itemId=" + itemId, e);
this.errorCount.incrementAndGet();
}
PriorityMQMessage item = queue.remove();
return item;
}
@Override
public PriorityMQMessage<E> take(long timeout) throws QueueInvalidStateException, QueueFatalException, QueueTimeoutException, DataStoreFatalException, InterruptedException {
checkIfShutdown();
if (timeout == 0) {
if (!lock.tryLock()) {
return null;
}
} else if (timeout > 0) {
if (!lock.tryLock(timeout, TimeUnit.MILLISECONDS)) {
throw new QueueTimeoutException("Timeout while waiting for lock [method=take(), queue=" + getName() + "]");
}
} else {
// wait forever until we get the lock
lock.lockInterruptibly();
}
try {
// - On TAKE
// - if the page TAKE succeeds, delete from page and DataStore
// - if the page TAKE fails AND "overflow"==true, load a page from DataStore
// (by using DataStoreIterator, constrained by pagesize)
// - if the page load from DataStore does not load all values, set "overflow"
// boolean value = false
if (queue.size() == 0 && overflow) { //maybe use memoryCount.get() instead of queue.size()? which is more performant?
int loaded = loadPage();
logger.trace("Page load {} items", loaded);
pageLoadedCount.incrementAndGet();
if (loaded < maxItemsInMemory) {
logger.trace("Reset overflow");
overflow = false;
}
overflowSize.addAndGet(loaded*-1); //subtract the number loaded from the overflowSize
memorySize.set(loaded);
}
try {
// continue waiting until an object is in the queue
while (queue.size() == 0) {
// if timeout is zero, then we're not supposed to wait
if (timeout == 0) {
return null;
} else if (timeout < 0) {
// wait indefinitely
notEmpty.await();
} else {
// FIXME: what about a "spurious wakeup" where the full timeout??
// await for a period of time (returns true if no timeout, false
// if there was a timeout
boolean waitTimeout = !notEmpty.await(timeout, TimeUnit.MILLISECONDS);
if (waitTimeout) {
throw new QueueTimeoutException("Timeout while waiting for item [method=take(), queue=" + getName() + "]");
}
}
}
} catch (InterruptedException ie) {
notEmpty.signal(); // propagate to non-interrupted thread
throw ie;
}
PriorityMQMessage<E> item = doTake();
size.decrementAndGet();
memorySize.decrementAndGet();
takeCount.incrementAndGet();
afterTake(item);
return item;
} finally {
lock.unlock();
}
}
/**
* Load a page from the ds. Call this only when the lock is acquired.
*/
private int loadPage() throws DataStoreFatalException {
int loaded = 0;
DataStoreIterator iterator = ds.getAscendingIterator();
iterator.jump(priorityKeyUtil.encode(getId(), 0L));
try {
do {
// get the next record
DataStoreIterator.Record record = iterator.getRecord();
byte[] k = record.getKey();
CompositeKey key = priorityKeyUtil.decode(k);
if (key.getQueueId() == getId()) {
queue.add(priorityTranscoder.decode(record.getValue()));
loaded++;
} else {
break;
}
} while (iterator.next() && loaded < maxItemsInMemory);
} finally {
iterator.close();
}
return loaded;
}
}