package com.cloudhopper.mq.queue.impl;
/*
* #%L
* ch-mq
* %%
* Copyright (C) 2012 Cloudhopper by Twitter
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import com.cloudhopper.commons.util.CircularIndex;
import com.cloudhopper.mq.queue.*;
import com.cloudhopper.mq.util.CompositeKey;
import com.cloudhopper.mq.util.CompositeKeyUtil;
import com.cloudhopper.datastore.DataStore;
import com.cloudhopper.datastore.DataStoreIterator;
import com.cloudhopper.datastore.DataStoreFatalException;
import com.cloudhopper.datastore.RecordNotFoundException;
import com.cloudhopper.mq.transcoder.Transcoder;
import java.util.LinkedList;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The original DefaultQueue that uses an in-memory LinkedList for all items, and persists
* everything to the DataStore in case of failure. Ported to be an Abstract, InitializingQueue.
*
* @author joelauer, garth
*/
public class DefaultQueue<E> extends AbstractQueue<E> implements Preloadable {
//public class DefaultQueue<E> extends BaseQueue<E> implements DefaultQueueMBean {
private static final Logger logger = LoggerFactory.getLogger(DefaultQueue.class);
// in-memory queue a simple linked list
private LinkedList<E> queue;
// circular index for adding/removing elements
private CircularIndex index;
/**
*/
public DefaultQueue() {
super();
}
protected void doActivate() throws Exception {
// default the queue to the backQueue
queue = backQueue;
// do we need to merge the front and back queues together?
if (frontQueue.size() > 0) {
// add the front part of the list first
queue = new LinkedList<E>(frontQueue);
// then add the back list
queue.addAll(backQueue);
}
if (firstItemId == -1) firstItemId = 0;
this.index = new CircularIndex(keyUtil.getMaxItems(), firstItemId, queue.size(), true);
this.size.set(queue.size());
this.putCount.set(queue.size());
}
private long firstItemId = -1L;
private long lastItemId = -1L;
private LinkedList<E> backQueue = new LinkedList<E>();
private LinkedList<E> frontQueue = new LinkedList<E>();
public void preload(DataStoreIterator iterator) throws DataStoreFatalException, QueueFatalException {
boolean useBackQueue = true;
while (iterator.next()) {
DataStoreIterator.Record record = iterator.getRecord();
CompositeKey key = keyUtil.decode(record.getKey());
// use the transcoder to decode the value
E element = null;
try {
element = transcoder.decode(record.getValue());
//logger.trace("preloaded {}:{}", key, element);
} catch (Throwable t) {
throw new QueueFatalException("Unable to decode element with transcoder for queueId " + getId() + ". Perhaps incorrect transcoder?", t);
}
// assume the first item is the first itemId
if (firstItemId < 0) {
firstItemId = key.getItemId();
} else {
// if this current itemId does not follow the lastItemId
// then this indicates we have a gap and this current itemId
// is really the first itemId
if (key.getItemId() > lastItemId+1) {
// this should only be allowed to happen once, if it happens
// more than once, then this indicates there is a gap of records
// that is incorrect and this means this queue is likely bad
if (!useBackQueue) {
throw new QueueFatalException("Gap of itemIds for queueId=" + getId() + ". Perhaps bad queue?");
}
// the current itemId is actually our first item
firstItemId = key.getItemId();
// start adding things to the "front" queue
useBackQueue = false;
} else if (key.getItemId() < lastItemId+1) {
throw new DataStoreFatalException("ItemId out of order while loading queue itemId=" + key.getItemId() + " < " + (lastItemId+1));
} else {
// key is in the correct order, no issue
}
}
if (useBackQueue) {
backQueue.add(element);
//logger.trace("added {} to backQueue", element);
} else {
frontQueue.add(element);
//logger.trace("added {} to frontQueue", element);
}
// set this lastItemId to the current itemId
lastItemId = key.getItemId();
}
}
protected boolean hasElements() {
return size.get() > 0;
}
protected byte[] encodeItem(E item) {
return transcoder.encode(item);
}
protected E decodeItem(byte[] encoded) {
return transcoder.decode(encoded);
}
protected boolean doStore(E item, byte[] encoded) throws QueueIsFullException {
// temporarily get the next index (but don't increment it yet)
long itemId = this.index.getNextLast();
logger.trace("[{}] itemId={} in queue.put()", getName(), itemId);
// is the queue full?
if (itemId < 0) {
throw new QueueIsFullException("Queue is full [method=add(), queue=" + getName() + ", size=" + index.getSize() + "]");
}
// generate the composite key we'll use within the queue (queueId and itemId)
byte[] key = keyUtil.encode(getId(), itemId);
// put the value into the data store by key -- will throw an exception
// if there was an error while storing it, this is the most likely
// place that an error would occur
try {
ds.setRecord(key, encoded);
} catch (DataStoreFatalException e) {
// this should only happen if there was a serious error with
// the underlying data store -- we'll ignore so that the system doesn't crash
logger.error("Unable to permanently store key and value for queueId=" + getId() + ", itemId=" + itemId, e);
this.errorCount.incrementAndGet();
return false;
}
queue.add(item);
return true;
}
protected void afterStore(E item, byte[] encoded) {
// if we got here, the item was added, increment the index
index.addLast();
}
protected E doTake() {
// temporarily get the first index (but don't increment it yet)
long itemId = this.index.getFirst();
logger.trace("[{}] itemId={} in queue.take()", getName(), itemId);
// something is drastically wrong if this is negative...
// FIXME: should we maybe throw an exception here?
/**
if (itemId < 0) {
//throw new QueueIsFullException("Queue is full [method=add(), queue=" + getName() + ", size=" + index.getSize() + "]");
}
*/
// generate the composite key we'll use within the queue (queueId and itemId)
byte[] key = keyUtil.encode(getId(), itemId);
// put the value into the data store by key -- will throw an exception
// if there was an error while storing it, this is the most likely
// place that an error would occur
try {
ds.deleteRecord(key);
} catch (RecordNotFoundException e) {
// this should only happen if there was a serious error with
// the underlying data store -- we'll ignore so that the system doesn't crash
logger.error("Key not found in DataStore for queueId=" + getId() + ", itemId=" + itemId, e);
this.errorCount.incrementAndGet();
} catch (DataStoreFatalException e) {
// this should only happen if there was a serious error with
// the underlying data store -- we'll ignore so that the system doesn't crash
logger.error("Unable to permanently delete key for queueId=" + getId() + ", itemId=" + itemId, e);
this.errorCount.incrementAndGet();
}
E item = queue.remove();
return item;
}
protected void afterTake(E item) {
// make sure the index is correct
this.index.removeFirst();
}
@Override
public E getMedianEntry(long timeout) throws QueueTimeoutException, QueueInvalidStateException, InterruptedException {
checkIfShutdown();
if (timeout == 0) {
if (!lock.tryLock()) {
throw new QueueTimeoutException("Lock not available [method=getMedianEntry(), queue=" + getName() + "]");
}
} else if (timeout > 0) {
if (!lock.tryLock(timeout, TimeUnit.MILLISECONDS)) {
throw new QueueTimeoutException("Timeout while waiting for lock [method=getMedianEntry(), queue=" + getName() + "]");
}
} else {
// wait forever until we get the lock
lock.lockInterruptibly();
}
try {
int size = queue.size();
if (size == 0)
return null;
// estimated; ceiling(size/2)
int median = size / 2;
E item = queue.get(median);
return item;
} finally {
lock.unlock();
}
}
}