package com.cloudhopper.mq.queue.impl;
/*
* #%L
* ch-mq
* %%
* Copyright (C) 2012 Cloudhopper by Twitter
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import com.cloudhopper.mq.message.PriorityMQMessage;
import com.cloudhopper.mq.queue.*;
import com.cloudhopper.mq.util.CompositeKeyUtil;
import com.cloudhopper.mq.util.CompositeKey;
import com.cloudhopper.mq.util.PriorityCompositeKeyUtil;
import com.cloudhopper.datastore.DataStore;
import com.cloudhopper.datastore.DataStoreFatalException;
import com.cloudhopper.datastore.DataStoreIterator;
import com.cloudhopper.mq.transcoder.PriorityMQMessageTranscoder;
import com.cloudhopper.mq.transcoder.Transcoder;
import java.util.LinkedList;
import java.util.Map;
import java.util.TreeMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Queue factory
* @author joelauer, garth
*/
public class DefaultQueueFactory implements QueueFactory {
private static final Logger logger = LoggerFactory.getLogger(DefaultQueueFactory.class);
private final DefaultQueueManager queueManager;
// for creating composite keys in the data store
private final CompositeKeyUtil keyUtil;
private final PriorityCompositeKeyUtil priorityKeyUtil;
// data store shared by persistent queues
private final DataStore ds;
public DefaultQueueFactory(DefaultQueueManager queueManager, CompositeKeyUtil keyUtil, DataStore ds) {
this.queueManager = queueManager;
this.keyUtil = keyUtil;
// PriorityCompositeKeyUtil has the same length queueId for backwards compatibility
this.priorityKeyUtil = new PriorityCompositeKeyUtil(keyUtil.getQueueIdByteLength(), 8);
this.ds = ds;
}
public <E> Queue<E> createQueue(Class<? extends InitializingQueue> queueType, Map<String,Object> properties, int queueId, String queueName, Class<E> elementType, Transcoder<E> transcoder) throws QueueFatalException, DataStoreFatalException {
try {
InitializingQueue iq = (InitializingQueue)queueType.newInstance();
iq.setQueueManager(queueManager);
iq.setQueueId(queueId);
iq.setQueueName(queueName);
iq.setLocalOnly(queueManager.isLocalOnly(queueName));
iq.setDataStore(ds);
iq.setCompositeKeyUtil(keyUtil);
iq.setTranscoder(transcoder);
iq.setElementType(elementType);
if (properties != null) {
for (Map.Entry<String,Object> prop : properties.entrySet()) {
iq.setProperty(prop.getKey(), prop.getValue());
}
}
iq.activate();
return (Queue<E>)iq;
} catch (Exception e) {
throw new QueueFatalException("Error creating queue "+queueName, e);
}
}
/**
* @deprecated As of version 2.1. Use createQueue that takes a Queue type instead.
*/
public <E> Queue<E> createQueue(int queueId, String queueName, Class<E> elementType, Transcoder<E> transcoder) throws QueueFatalException, DataStoreFatalException {
return createQueue(DefaultQueue.class, null, queueId, queueName, elementType, transcoder);
}
/**
* @deprecated As of version 2.1. Use createQueue that takes a Queue type instead.
*/
public <E> Queue<PriorityMQMessage<E>> createPriorityQueue(int queueId, String queueName, Class<E> elementType, Transcoder<E> transcoder) throws QueueFatalException, DataStoreFatalException {
return (Queue<PriorityMQMessage<E>>)createQueue(DefaultPriorityQueue.class, null, queueId, queueName, elementType, transcoder);
}
@SuppressWarnings("unchecked")
public TreeMap<Integer,Queue> loadQueues(TreeMap<Integer,QueueInfo> queueInfoMap) throws QueueFatalException, DataStoreFatalException {
return loadInitializingQueues(queueInfoMap);
}
@SuppressWarnings("unchecked")
public TreeMap<Integer,Queue> loadInitializingQueues(TreeMap<Integer,QueueInfo> queueInfoMap) throws QueueFatalException, DataStoreFatalException {
// map of queueIds to queue
TreeMap<Integer,Queue> queueIdMap = new TreeMap<Integer,Queue>();
// Initialize the queues in the map
for (Map.Entry<Integer,QueueInfo> entry : queueInfoMap.entrySet()) {
try {
Queue q = (Queue)entry.getValue().getQueueType().newInstance();
if (q instanceof InitializingQueue) {
InitializingQueue iq = (InitializingQueue)q;
iq.setQueueManager(queueManager);
iq.setQueueId(entry.getKey().intValue());
iq.setQueueName(entry.getValue().getQueueName());
iq.setLocalOnly(queueManager.isLocalOnly(entry.getValue().getQueueName()));
iq.setDataStore(ds);
iq.setCompositeKeyUtil(keyUtil);
iq.setTranscoder((Transcoder)entry.getValue().getTranscoderType().newInstance());
iq.setElementType(entry.getValue().getElementType());
//iq.setProperty(String name, Object value);
}
queueIdMap.put(entry.getKey(), q);
} catch (Exception e) {
throw new QueueFatalException("Error initializing queue", e);
}
}
// iterate thru all the current data in the data store by ascending key
DataStoreIterator iterator = ds.getAscendingIterator();
try {
if (iterator != null && iterator.next()) { //the iterator exists, and has at least 1 record
int queueId = -1;
QueueInfo queueInfo = null;
Queue queue = null;
while (true) {
queueId = moveToNextQueue(iterator, queueId);
if (queueId < 0) break; //there are no more queues
queueInfo = queueInfoMap.get(queueId);
logger.trace("Current queue at {} is {}", queueId, queueInfo);
if (queueInfo == null) {
throw new QueueFatalException("The queueId " + queueId + " was in the DataStore, but was not in our queueInfoMap");
}
queue = queueIdMap.get(queueId);
logger.info("Loading persistent queue with id=" + queueId);
if (queue instanceof Preloadable) {
try {
((Preloadable)queue).preload(new PerQueueDataStoreIterator(queueId, priorityKeyUtil.getQueueIdByteLength(), iterator));
} catch (Exception e) {
throw new QueueFatalException("Error preloading queue "+queueId, e);
}
}
}
}
} finally {
iterator.close();
}
// Activate the queues
for (Map.Entry<Integer,Queue> entry : queueIdMap.entrySet()) {
if (entry.getValue() instanceof InitializingQueue) {
InitializingQueue iq = (InitializingQueue)entry.getValue();
try {
iq.activate();
logger.trace("Activated queue {}", iq);
} catch (Exception e) {
throw new QueueFatalException("Error activating queue", e);
}
}
}
return queueIdMap;
}
/**
* Moves the iterator to the next queue that isn't the given currentQueueId.
*/
private int moveToNextQueue(DataStoreIterator iterator, int currentQueueId) throws DataStoreFatalException {
do {
try {
DataStoreIterator.Record record = iterator.getRecord();
CompositeKey key = priorityKeyUtil.decode(record.getKey());
if (key.getQueueId() != currentQueueId) return key.getQueueId();
} catch (DataStoreFatalException e) {
//we might not be on the first record?
}
} while (iterator.next());
return -1;
}
/*
@SuppressWarnings("unchecked")
public TreeMap<Integer,Queue> loadQueues(TreeMap<Integer,QueueInfo> queueInfoMap) throws QueueFatalException, DataStoreFatalException {
// map of queueIds to queue
TreeMap<Integer,Queue> queueIdMap = new TreeMap<Integer,Queue>();
//
// first, we'll create any queues that have data persisted
//
// iterate thru all the current data in the data store by ascending key
DataStoreIterator iterator = ds.getAscendingIterator();
try {
String queueName = null;
int queueId = -1;
boolean circularQueue = false;
Transcoder transcoder = null;
Class queueType = null;
Class elementType = null;
long firstItemId = -1;
long lastItemId = -1;
LinkedList backQueue = null;
LinkedList frontQueue = null;
boolean useBackQueue = true;
QueueInfo queueInfo = null;
while (iterator.next()) {
// get the next record
DataStoreIterator.Record record = iterator.getRecord();
// decode the record's key
// CompositeKey key = keyUtil.decode(record.getKey());
CompositeKey key = priorityKeyUtil.decode(record.getKey());
// does this record represent the start of a new queue?
if (key.getQueueId() != queueId) {
// reset priority
circularQueue = false;
// if this isn't the first queue, then backQueue won't be null
// this means we need to save the previously loaded queue
if (backQueue != null) {
//OLD createAndAddQueue(queueIdMap, queueId, queueName, transcoder, elementType, firstItemId, backQueue, frontQueue);
createAndAddQueue(queueIdMap, queueInfo, queueId, queueName, transcoder, elementType, firstItemId, backQueue, frontQueue);
}
// set the new queueId as our current queueId
queueId = key.getQueueId();
// this queueId MUST exist in our queueInfoMap
queueInfo = queueInfoMap.get(queueId);
logger.trace("Current queue at {} is {}", queueId, queueInfo);
if (queueInfo == null) {
throw new QueueFatalException("The queueId " + queueId + " was in the DataStore, but was not in our queueInfoMap");
}
queueName = queueInfo.getQueueName();
queueType = queueInfo.getQueueType();
elementType = queueInfo.getElementType();
// the queueInfo object will help us create the correct transcoder
try {
if (queueType.equals(DefaultQueue.class)) {
transcoder = (Transcoder)queueInfo.getTranscoderType().newInstance();
circularQueue = true;
} else if (queueType.equals(DefaultPriorityQueue.class)) {
Transcoder t = (Transcoder)queueInfo.getTranscoderType().newInstance();
transcoder = new PriorityMQMessageTranscoder(t);
circularQueue = false;
} else {
throw new RuntimeException("Unknown queue type "+queueType.getName());
}
} catch (Exception e) {
throw new QueueFatalException("Unable to create transcoderType for queueId " + queueId, e);
}
// reset the other things we'll load for this queue
backQueue = new LinkedList();
frontQueue = new LinkedList();
useBackQueue = true;
// reset the first and last itemId
firstItemId = -1;
lastItemId = -1;
logger.info("Loading persistent queue name=" + queueName + " with id=" + queueId);
}
// use the transcoder to decode the value
Object element = null;
try {
element = transcoder.decode(record.getValue());
} catch (Throwable t) {
throw new QueueFatalException("Unable to decode element with transcoder for queueId " + queueId + ". Perhaps incorrect transcoder?", t);
}
logger.trace("Circular Queue = {}", circularQueue);
if (circularQueue) {
// assume the first item is the first itemId
if (firstItemId < 0) {
firstItemId = key.getItemId();
} else {
// if this current itemId does not follow the lastItemId
// then this indicates we have a gap and this current itemId
// is really the first itemId
if (key.getItemId() > lastItemId+1) {
// this should only be allowed to happen once, if it happens
// more than once, then this indicates there is a gap of records
// that is incorrect and this means this queue is likely bad
if (!useBackQueue) {
throw new QueueFatalException("Gap of itemIds for queueId=" + queueId + ". Perhaps bad queue?");
}
// the current itemId is actually our first item
firstItemId = key.getItemId();
// start adding things to the "front" queue
useBackQueue = false;
} else if (key.getItemId() < lastItemId+1) {
throw new DataStoreFatalException("ItemId out of order while loading queue itemId=" + key.getItemId() + " < " + (lastItemId+1));
} else {
// key is in the correct order, no issue
}
}
}
if (useBackQueue) {
backQueue.add(element);
} else {
frontQueue.add(element);
}
// set this lastItemId to the current itemId
lastItemId = key.getItemId();
}
// handle either none, the first, or the last queue in our data store...
if (backQueue != null) {
createAndAddQueue(queueIdMap, queueInfo, queueId, queueName, transcoder, elementType, firstItemId, backQueue, frontQueue);
}
} finally {
iterator.close();
}
// persistent queues are now loaded, but there may still be queues
// that were in the QueueInfo map, but didn't have any items in the
// DataStore -- these are ones we'll create empty queues for
for (Map.Entry<Integer,QueueInfo> entry : queueInfoMap.entrySet()) {
// is this queue missing?
Integer queueId = entry.getKey();
QueueInfo queueInfo = entry.getValue();
if (!queueIdMap.containsKey(queueId)) {
// the queueInfo object will help us create the correct transcoder
Transcoder transcoder = null;
try {
if (queueInfo.getQueueType().equals(DefaultQueue.class)) {
transcoder = (Transcoder)queueInfo.getTranscoderType().newInstance();
} else if (queueInfo.getQueueType().equals(DefaultPriorityQueue.class)) {
Transcoder t = (Transcoder)queueInfo.getTranscoderType().newInstance();
transcoder = new PriorityMQMessageTranscoder(t);
} else {
throw new RuntimeException("Unknown queue type "+queueInfo.getQueueType().getName());
}
} catch (Exception e) {
throw new QueueFatalException("Unable to create transcoderType for queueId " + queueId, e);
}
// create a new queue and add it
Queue queue = null;
if (queueInfo.getQueueType().equals(DefaultQueue.class)) {
queue = this.createQueue(queueId, queueInfo.getQueueName(), queueInfo.getElementType(), transcoder);
} else if (queueInfo.getQueueType().equals(DefaultPriorityQueue.class)) {
queue = this.createPriorityQueue(queueId, queueInfo.getQueueName(), queueInfo.getElementType(), transcoder);
} else {
throw new RuntimeException("Unknown queue type "+queueInfo.getQueueType().getName());
}
queueIdMap.put(queueId, queue);
}
}
return queueIdMap;
}
*/
/*
@SuppressWarnings("unchecked") private void createAndAddQueue(TreeMap<Integer,Queue> queueIdMap, QueueInfo queueInfo, int queueId, String queueName, Transcoder transcoder, Class elementType, long firstItemId, LinkedList backQueue, LinkedList frontQueue) throws QueueFatalException, DataStoreFatalException {
// default the internalQueue to the backQueue
LinkedList internalQueue = backQueue;
// do we need to merge the front and back queues together?
if (frontQueue.size() > 0) {
// add the front part of the list first
internalQueue = new LinkedList(frontQueue);
// then add the back list
internalQueue.addAll(backQueue);
}
Queue queue = null;
if (queueInfo.getQueueType().equals(DefaultQueue.class)) {
queue = createQueue(DefaultQueue.class, null, queueId, queueName, elementType, transcoder);
} else if (queueInfo.getQueueType().equals(DefaultPriorityQueue.class)) {
queue = createQueue(DefaultPriorityQueue.class, null, queueId, queueName, elementType, transcoder);
} else {
throw new RuntimeException("Unknown queue type "+queueInfo.getQueueType().getName());
}
// add this queue to the map
queueIdMap.put(queueId, queue);
}
*/
}