package com.cloudhopper.mq.queue.impl;
/*
* #%L
* ch-mq
* %%
* Copyright (C) 2012 Cloudhopper by Twitter
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import com.cloudhopper.mq.message.PriorityMQMessage;
import com.cloudhopper.mq.queue.*;
import com.cloudhopper.mq.util.CompositeKey;
import com.cloudhopper.mq.util.CompositeKeyUtil;
import com.cloudhopper.mq.util.PriorityCompositeKeyUtil;
import com.cloudhopper.datastore.DataStore;
import com.cloudhopper.datastore.DataStoreIterator;
import com.cloudhopper.datastore.DataStoreFatalException;
import com.cloudhopper.datastore.RecordNotFoundException;
import com.cloudhopper.mq.transcoder.PriorityMQMessageTranscoder;
import com.cloudhopper.mq.transcoder.Transcoder;
import java.util.PriorityQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* @author garth
*/
public class DefaultPriorityQueue<E> extends AbstractQueue<PriorityMQMessage<E>> implements Preloadable {
private static final Logger logger = LoggerFactory.getLogger(DefaultPriorityQueue.class);
// in-memory priority queue
private final PriorityQueue<PriorityMQMessage> queue = new PriorityQueue<PriorityMQMessage>();;
protected PriorityCompositeKeyUtil priorityKeyUtil;
protected Transcoder<E> baseTranscoder;
protected Transcoder<PriorityMQMessage<E>> priorityTranscoder;
public DefaultPriorityQueue() {
super();
}
protected void doActivate() throws Exception {
this.size.set(queue.size());
this.putCount.set(queue.size());
}
public void preload(DataStoreIterator iterator) throws DataStoreFatalException, QueueFatalException {
while (iterator.next()) {
DataStoreIterator.Record record = iterator.getRecord();
CompositeKey key = priorityKeyUtil.decode(record.getKey());
PriorityMQMessage<E> element = null;
try {
element = priorityTranscoder.decode(record.getValue());
} catch (Throwable t) {
throw new QueueFatalException("Unable to decode element with transcoder for queueId " + getId() + ". Perhaps incorrect transcoder?", t);
}
nextSequence(element);
this.queue.add(element);
}
}
@Override
public void setCompositeKeyUtil(CompositeKeyUtil keyUtil) {
super.setCompositeKeyUtil(keyUtil);
this.priorityKeyUtil = new PriorityCompositeKeyUtil(keyUtil.getQueueIdByteLength(), 8);
}
@Override
public void setTranscoder(Transcoder transcoder) {
this.baseTranscoder = (Transcoder<E>)transcoder;
this.priorityTranscoder = new PriorityMQMessageTranscoder<E>(transcoder);
}
@Override
public Transcoder<PriorityMQMessage<E>> getTranscoder() {
return this.priorityTranscoder;
}
@Override
public Class<?> getTranscoderType() {
return this.baseTranscoder.getClass();
}
@Override
public String getTranscoderTypeName() {
return this.baseTranscoder.getClass().getCanonicalName();
}
protected boolean hasElements() {
return size.get() > 0;
}
protected byte[] encodeItem(PriorityMQMessage<E> item) {
nextSequence(item);
return priorityTranscoder.encode(item);
}
protected PriorityMQMessage<E> decodeItem(byte[] encoded) {
return priorityTranscoder.decode(encoded);
}
@Override protected boolean doStore(PriorityMQMessage<E> item, byte[] encoded) throws QueueInvalidStateException, QueueFatalException, QueueIsFullException, QueueTimeoutException, DataStoreFatalException {
long itemId = item.key();
byte[] key = priorityKeyUtil.encode(getId(), itemId);
try {
ds.setRecord(key, encoded);
} catch (DataStoreFatalException e) {
logger.error("Unable to permanently store key and value for queueId=" + getId() + ", itemId=" + itemId, e);
this.errorCount.incrementAndGet();
}
return queue.add(item);
}
@Override protected void afterStore(PriorityMQMessage<E> item, byte[] encoded) {
// nothing
}
@Override protected PriorityMQMessage<E> doTake() throws QueueInvalidStateException, QueueFatalException, QueueTimeoutException, DataStoreFatalException {
PriorityMQMessage w = queue.peek();
long itemId = w.key();
byte[] key = priorityKeyUtil.encode(getId(), itemId);
try {
ds.deleteRecord(key);
} catch (RecordNotFoundException e) {
logger.error("Key not found in DataStore for queueId=" + getId() + ", itemId=" + itemId, e);
this.errorCount.incrementAndGet();
} catch (DataStoreFatalException e) {
logger.error("Unable to permanently delete key for queueId=" + getId() + ", itemId=" + itemId, e);
this.errorCount.incrementAndGet();
}
PriorityMQMessage item = queue.remove();
return item;
}
@Override protected void afterTake(PriorityMQMessage<E> item) {
// nothing
}
private short sequence = 0;
private long ts = System.currentTimeMillis();;
/**
* Set the ts and sequence. Return the itemId.
*/
synchronized protected long nextSequence(PriorityMQMessage message) {
long now = System.currentTimeMillis();
if (ts == now) sequence++;
else sequence = 0;
if (sequence > 255) {
logger.warn("Sequence rollover for {}", this);
try {
Thread.sleep(1);
} catch (InterruptedException e) {}
return nextSequence(message);
}
//TODO: need to rethink this. allowing external setting of the sequence means that we may have
// colliding keys (e.g. when decoding messages from 2 different remote brokers)
// For now, we are just ignoring externally set sequence and timestamp when we go to create the key.
// if (!message.isSequenceSet()) message.setSequence(sequence);
// if (!message.isTimestampSet()) message.setTimestamp(now);
message.setSequence(sequence);
message.setTimestamp(now);
ts = now;
return message.key();
}
}