package com.cloudhopper.mq.queue.impl;
/*
* #%L
* ch-mq
* %%
* Copyright (C) 2012 Cloudhopper by Twitter
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import com.cloudhopper.commons.util.CircularIndex;
import com.cloudhopper.mq.queue.*;
import com.cloudhopper.mq.util.CompositeKey;
import com.cloudhopper.mq.util.CompositeKeyUtil;
import com.cloudhopper.datastore.DataStore;
import com.cloudhopper.datastore.DataStoreIterator;
import com.cloudhopper.datastore.DataStoreFatalException;
import com.cloudhopper.datastore.RecordNotFoundException;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* A Queue that uses the DataSource directly with no in-memory buffering.
*
* @author garth
*/
public class DirectQueue<E> extends AbstractQueue<E> implements Preloadable {
private static final Logger logger = LoggerFactory.getLogger(DirectQueue.class);
// circular index for adding/removing elements
private CircularIndex index;
/**
*/
public DirectQueue() {
super();
}
protected void doActivate() throws Exception {
if (firstItemId == -1) firstItemId = 0;
this.index = new CircularIndex(keyUtil.getMaxItems(), firstItemId, queueSize, true);
this.size.set(queueSize);
this.putCount.set(queueSize);
}
private long firstItemId = -1L;
private long lastItemId = -1L;
private long queueSize = 0L;
public void preload(DataStoreIterator iterator) throws DataStoreFatalException, QueueFatalException {
while (iterator.next()) {
DataStoreIterator.Record record = iterator.getRecord();
CompositeKey key = keyUtil.decode(record.getKey());
queueSize++;
// assume the first item is the first itemId
if (firstItemId < 0) {
firstItemId = key.getItemId();
} else {
// if this current itemId does not follow the lastItemId
// then this indicates we have a gap and this current itemId
// is really the first itemId
if (key.getItemId() > lastItemId + 1) {
// the current itemId is actually our first item
firstItemId = key.getItemId();
} else if (key.getItemId() < lastItemId + 1) {
throw new DataStoreFatalException("ItemId out of order while loading queue itemId=" + key.getItemId() + " < " + (lastItemId + 1));
} else {
// key is in the correct order, no issue
}
}
// set this lastItemId to the current itemId
lastItemId = key.getItemId();
}
}
protected boolean hasElements() {
return size.get() > 0;
}
protected byte[] encodeItem(E item) {
return transcoder.encode(item);
}
protected E decodeItem(byte[] encoded) {
return transcoder.decode(encoded);
}
protected boolean doStore(E item, byte[] encoded) throws QueueIsFullException {
// temporarily get the next index (but don't increment it yet)
long itemId = this.index.getNextLast();
logger.trace("[{}] itemId={} in queue.put()", getName(), itemId);
// is the queue full?
if (itemId < 0) {
throw new QueueIsFullException("Queue is full [method=add(), queue=" + getName() + ", size=" + index.getSize() + "]");
}
// generate the composite key we'll use within the queue (queueId and itemId)
byte[] key = keyUtil.encode(getId(), itemId);
// put the value into the data store by key -- will throw an exception
// if there was an error while storing it, this is the most likely
// place that an error would occur
try {
ds.setRecord(key, encoded);
} catch (DataStoreFatalException e) {
// this should only happen if there was a serious error with
// the underlying data store -- we'll ignore so that the system doesn't crash
logger.error("Unable to permanently store key and value for queueId=" + getId() + ", itemId=" + itemId, e);
this.errorCount.incrementAndGet();
return false;
}
return true;
}
protected void afterStore(E item, byte[] encoded) {
// if we got here, the item was added, increment the index
index.addLast();
}
protected E doTake() {
// temporarily get the first index (but don't increment it yet)
long itemId = this.index.getFirst();
logger.trace("[{}] itemId={} in queue.take()", getName(), itemId);
// something is drastically wrong if this is negative...
// FIXME: should we maybe throw an exception here?
/**
if (itemId < 0) {
//throw new QueueIsFullException("Queue is full [method=add(), queue=" + getName() + ", size=" + index.getSize() + "]");
}
*/
// generate the composite key we'll use within the queue (queueId and itemId)
byte[] key = keyUtil.encode(getId(), itemId);
E item = null;
// put the value into the data store by key -- will throw an exception
// if there was an error while storing it, this is the most likely
// place that an error would occur
try {
byte[] encoded = ds.getRecord(key);
item = decodeItem(encoded);
ds.deleteRecord(key);
} catch (RecordNotFoundException e) {
// this should only happen if there was a serious error with
// the underlying data store -- we'll ignore so that the system doesn't crash
logger.error("Key not found in DataStore for queueId=" + getId() + ", itemId=" + itemId, e);
this.errorCount.incrementAndGet();
} catch (DataStoreFatalException e) {
// this should only happen if there was a serious error with
// the underlying data store -- we'll ignore so that the system doesn't crash
logger.error("Unable to permanently delete key for queueId=" + getId() + ", itemId=" + itemId, e);
this.errorCount.incrementAndGet();
}
return item;
}
protected void afterTake(E item) {
// make sure the index is correct
this.index.removeFirst();
}
//TESTING. Don't use.
public E peek(long timeout) throws QueueInvalidStateException, QueueFatalException, QueueTimeoutException, DataStoreFatalException, InterruptedException {
checkIfShutdown();
if (timeout == 0) {
if (!lock.tryLock()) {
return null;
}
} else if (timeout > 0) {
if (!lock.tryLock(timeout, TimeUnit.MILLISECONDS)) {
throw new QueueTimeoutException("Timeout while waiting for lock [method=take(), queue=" + getName() + "]");
}
} else {
// wait forever until we get the lock
lock.lockInterruptibly();
}
try {
try {
// continue waiting until an object is in the queue
//while (queue.size() == 0) {
while (!hasElements()) {
// if timeout is zero, then we're not supposed to wait
if (timeout == 0) {
return null;
} else if (timeout < 0) {
// wait indefinitely
notEmpty.await();
} else {
// FIXME: what about a "spurious wakeup" where the full timeout??
// await for a period of time (returns true if no timeout, false
// if there was a timeout
boolean waitTimeout = !notEmpty.await(timeout, TimeUnit.MILLISECONDS);
if (waitTimeout) {
throw new QueueTimeoutException("Timeout while waiting for item [method=take(), queue=" + getName() + "]");
}
}
}
} catch (InterruptedException ie) {
notEmpty.signal(); // propagate to non-interrupted thread
throw ie;
}
E item = null;
//E item = doTake();
/////////////////////////
// temporarily get the first index (but don't increment it yet)
long itemId = this.index.getFirst();
logger.trace("[{}] itemId={} in queue.take()", getName(), itemId);
// something is drastically wrong if this is negative...
// FIXME: should we maybe throw an exception here?
/**
if (itemId < 0) {
//throw new QueueIsFullException("Queue is full [method=add(), queue=" + getName() + ", size=" + index.getSize() + "]");
}
*/
// generate the composite key we'll use within the queue (queueId and itemId)
byte[] key = keyUtil.encode(getId(), itemId);
// put the value into the data store by key -- will throw an exception
// if there was an error while storing it, this is the most likely
// place that an error would occur
try {
byte[] encoded = ds.getRecord(key);
item = decodeItem(encoded);
} catch (RecordNotFoundException e) {
// this should only happen if there was a serious error with
// the underlying data store -- we'll ignore so that the system doesn't crash
logger.error("Key not found in DataStore for queueId=" + getId() + ", itemId=" + itemId, e);
this.errorCount.incrementAndGet();
} catch (DataStoreFatalException e) {
// this should only happen if there was a serious error with
// the underlying data store -- we'll ignore so that the system doesn't crash
logger.error("Unable to permanently delete key for queueId=" + getId() + ", itemId=" + itemId, e);
this.errorCount.incrementAndGet();
}
/////////////////////////
// TODO: should this happen after afterTake(item)
//size.decrementAndGet();
//takeCount.incrementAndGet();
//afterTake(item);
// mark the take rate meter
//takeRateMeter.mark();
return item;
} finally {
lock.unlock();
}
}
}