/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.activemq.broker.region.cursors;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import javax.jms.Connection;
import javax.jms.ConnectionFactory;
import javax.jms.Destination;
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.MessageConsumer;
import javax.jms.MessageListener;
import javax.jms.MessageProducer;
import javax.jms.Queue;
import javax.jms.Session;
import javax.jms.TextMessage;
import javax.management.MalformedObjectNameException;
import javax.management.ObjectName;
import org.apache.activemq.ActiveMQConnectionFactory;
import org.apache.activemq.AutoFailTestSupport;
import org.apache.activemq.broker.BrokerService;
import org.apache.activemq.broker.jmx.QueueViewMBean;
import org.apache.activemq.broker.region.policy.PolicyEntry;
import org.apache.activemq.broker.region.policy.PolicyMap;
import org.apache.activemq.broker.region.policy.StorePendingQueueMessageStoragePolicy;
import org.apache.activemq.usage.MemoryUsage;
import org.apache.activemq.usage.StoreUsage;
import org.apache.activemq.usage.SystemUsage;
import org.apache.activemq.usage.TempUsage;
import org.apache.activemq.util.Wait;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Modified CursorSupport Unit test to reproduce the negative queue issue.
*
* Keys to reproducing:
* 1) Consecutive queues with listener on first sending to second queue
* 2) Push each queue to the memory limit
* This seems to help reproduce the issue more consistently, but
* we have seen times in our production environment where the
* negative queue can occur without. Our memory limits are
* very high in production and it still happens in varying
* frequency.
* 3) Prefetch
* Lowering the prefetch down to 10 and below seems to help
* reduce occurrences.
* 4) # of consumers per queue
* The issue occurs less with fewer consumers
*
* Things that do not affect reproduction:
* 1) Spring - we use spring in our production applications, but this test case works
* with or without it.
* 2) transacted
*
*/
public class NegativeQueueTest extends AutoFailTestSupport {
private static final Logger LOG = LoggerFactory.getLogger(NegativeQueueTest.class);
public static SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd,hh:mm:ss:SSS");
private static final String QUEUE_1_NAME = "conn.test.queue.1";
private static final String QUEUE_2_NAME = "conn.test.queue.2";
private static final long QUEUE_MEMORY_LIMIT = 2097152;
private static final long MEMORY_USAGE = 400000000;
private static final long TEMP_USAGE = 200000000;
private static final long STORE_USAGE = 1000000000;
private static final int MESSAGE_COUNT = 1100;
protected static final boolean TRANSACTED = true;
protected static final boolean DEBUG = true;
protected static int NUM_CONSUMERS = 20;
protected static int PREFETCH_SIZE = 1000;
protected BrokerService broker;
protected String bindAddress = "tcp://localhost:0";
public void testWithDefaultPrefetch() throws Exception{
PREFETCH_SIZE = 1000;
NUM_CONSUMERS = 20;
blastAndConsume();
}
public void x_testWithDefaultPrefetchFiveConsumers() throws Exception{
PREFETCH_SIZE = 1000;
NUM_CONSUMERS = 5;
blastAndConsume();
}
public void x_testWithDefaultPrefetchTwoConsumers() throws Exception{
PREFETCH_SIZE = 1000;
NUM_CONSUMERS = 2;
blastAndConsume();
}
public void testWithDefaultPrefetchOneConsumer() throws Exception{
PREFETCH_SIZE = 1000;
NUM_CONSUMERS = 1;
blastAndConsume();
}
public void testWithMediumPrefetch() throws Exception{
PREFETCH_SIZE = 50;
NUM_CONSUMERS = 20;
blastAndConsume();
}
public void x_testWithSmallPrefetch() throws Exception{
PREFETCH_SIZE = 10;
NUM_CONSUMERS = 20;
blastAndConsume();
}
public void testWithNoPrefetch() throws Exception{
PREFETCH_SIZE = 1;
NUM_CONSUMERS = 20;
blastAndConsume();
}
public void blastAndConsume() throws Exception {
LOG.info(getName());
ConnectionFactory factory = createConnectionFactory();
//get proxy queues for statistics lookups
Connection proxyConnection = factory.createConnection();
proxyConnection.start();
Session proxySession = proxyConnection.createSession(false, Session.AUTO_ACKNOWLEDGE);
final QueueViewMBean proxyQueue1 = getProxyToQueueViewMBean(proxySession.createQueue(QUEUE_1_NAME));
final QueueViewMBean proxyQueue2 = getProxyToQueueViewMBean(proxySession.createQueue(QUEUE_2_NAME));
// LOAD THE QUEUE
Connection producerConnection = factory.createConnection();
producerConnection.start();
Session session = producerConnection.createSession(TRANSACTED, Session.AUTO_ACKNOWLEDGE);
Destination queue = session.createQueue(QUEUE_1_NAME);
MessageProducer producer = session.createProducer(queue);
List<TextMessage> senderList = new ArrayList<TextMessage>();
for (int i = 0; i < MESSAGE_COUNT; i++) {
TextMessage msg = session.createTextMessage(i + " " + formatter.format(new Date()));
senderList.add(msg);
producer.send(msg);
if(TRANSACTED) session.commit();
if(DEBUG && i%100 == 0){
int index = (i/100)+1;
System.out.print(index-((index/10)*10));
}
}
//get access to the Queue info
if(DEBUG){
System.out.println("");
System.out.println("Queue1 Size = "+proxyQueue1.getQueueSize());
System.out.println("Queue1 Memory % Used = "+proxyQueue1.getMemoryPercentUsage());
System.out.println("Queue1 Memory Available = "+proxyQueue1.getMemoryLimit());
}
// FLUSH THE QUEUE
final CountDownLatch latch1 = new CountDownLatch(1);
final CountDownLatch latch2 = new CountDownLatch(1);
Connection[] consumerConnections1 = new Connection[NUM_CONSUMERS];
List<Message> consumerList1 = new ArrayList<Message>();
Connection[] consumerConnections2 = new Connection[NUM_CONSUMERS];
Connection[] producerConnections2 = new Connection[NUM_CONSUMERS];
List<Message> consumerList2 = new ArrayList<Message>();
for(int ix=0; ix<NUM_CONSUMERS; ix++){
producerConnections2[ix] = factory.createConnection();
producerConnections2[ix].start();
consumerConnections1[ix] = getConsumerConnection(factory);
Session consumerSession = consumerConnections1[ix].createSession(TRANSACTED, Session.AUTO_ACKNOWLEDGE);
MessageConsumer consumer = consumerSession.createConsumer(session.createQueue(QUEUE_1_NAME));
consumer.setMessageListener(new SessionAwareMessageListener(producerConnections2[ix], consumerSession, QUEUE_2_NAME, latch1, consumerList1));
}
latch1.await(200000, TimeUnit.MILLISECONDS);
if(DEBUG){
System.out.println("");
System.out.println("Queue2 Size = "+proxyQueue2.getQueueSize());
System.out.println("Queue2 Memory % Used = "+proxyQueue2.getMemoryPercentUsage());
System.out.println("Queue2 Memory Available = "+proxyQueue2.getMemoryLimit());
}
for(int ix=0; ix<NUM_CONSUMERS; ix++){
consumerConnections2[ix] = getConsumerConnection(factory);
Session consumerSession = consumerConnections2[ix].createSession(TRANSACTED, Session.AUTO_ACKNOWLEDGE);
MessageConsumer consumer = consumerSession.createConsumer(session.createQueue(QUEUE_2_NAME));
consumer.setMessageListener(new SessionAwareMessageListener(consumerSession, latch2, consumerList2));
}
boolean success = Wait.waitFor(new Wait.Condition() {
@Override
public boolean isSatisified() throws Exception {
boolean done = latch2.await(10, TimeUnit.SECONDS);
if(DEBUG){
System.out.println("");
System.out.println("Queue1 Size = "+proxyQueue1.getQueueSize());
System.out.println("Queue1 Memory % Used = "+proxyQueue1.getMemoryPercentUsage());
System.out.println("Queue2 Size = "+proxyQueue2.getQueueSize());
System.out.println("Queue2 Memory % Used = "+proxyQueue2.getMemoryPercentUsage());
System.out.println("Queue2 Memory Available = "+proxyQueue2.getMemoryLimit());
}
return done;
}
}, 300 * 1000);
if (!success) {
dumpAllThreads("blocked waiting on 2");
}
assertTrue("got all expected messages on 2", success);
producerConnection.close();
for(int ix=0; ix<NUM_CONSUMERS; ix++){
consumerConnections1[ix].close();
consumerConnections2[ix].close();
producerConnections2[ix].close();
}
//let the consumer statistics on queue2 have time to update
Thread.sleep(500);
if(DEBUG){
System.out.println("");
System.out.println("Queue1 Size = "+proxyQueue1.getQueueSize());
System.out.println("Queue1 Memory % Used = "+proxyQueue1.getMemoryPercentUsage());
System.out.println("Queue2 Size = "+proxyQueue2.getQueueSize());
System.out.println("Queue2 Memory % Used = "+proxyQueue2.getMemoryPercentUsage());
}
Wait.waitFor(new Wait.Condition() {
@Override
public boolean isSatisified() throws Exception {
return 0 == proxyQueue1.getQueueSize();
}});
assertEquals("Queue1 has gone negative,",0, proxyQueue1.getQueueSize());
Wait.waitFor(new Wait.Condition() {
@Override
public boolean isSatisified() throws Exception {
return 0 == proxyQueue2.getQueueSize();
}});
assertEquals("Queue2 has gone negative,",0, proxyQueue2.getQueueSize());
proxyConnection.close();
}
private QueueViewMBean getProxyToQueueViewMBean(Queue queue) throws MalformedObjectNameException, JMSException {
final String prefix = "org.apache.activemq:type=Broker,brokerName=localhost,destinationType=Queue,destinationName=";
ObjectName queueViewMBeanName = new ObjectName(prefix + queue.getQueueName());
QueueViewMBean proxy = (QueueViewMBean)
broker.getManagementContext().newProxyInstance(queueViewMBeanName, QueueViewMBean.class, true);
return proxy;
}
protected Connection getConsumerConnection(ConnectionFactory fac) throws JMSException {
Connection connection = fac.createConnection();
connection.start();
return connection;
}
@Override
protected void setUp() throws Exception {
if (broker == null) {
broker = createBroker();
}
super.setUp();
}
@Override
protected void tearDown() throws Exception {
super.tearDown();
if (broker != null) {
broker.stop();
broker.waitUntilStopped();
}
}
protected ActiveMQConnectionFactory createConnectionFactory() throws Exception {
ActiveMQConnectionFactory cf = new ActiveMQConnectionFactory(bindAddress);
Properties props = new Properties();
props.setProperty("prefetchPolicy.durableTopicPrefetch", "" + PREFETCH_SIZE);
props.setProperty("prefetchPolicy.optimizeDurableTopicPrefetch", "" + PREFETCH_SIZE);
props.setProperty("prefetchPolicy.queuePrefetch", "" + PREFETCH_SIZE);
cf.setProperties(props);
return cf;
}
protected BrokerService createBroker() throws Exception {
BrokerService answer = new BrokerService();
configureBroker(answer);
answer.start();
answer.waitUntilStarted();
bindAddress = answer.getTransportConnectors().get(0).getConnectUri().toString();
return answer;
}
protected void configureBroker(BrokerService answer) throws Exception {
PolicyEntry policy = new PolicyEntry();
policy.setMemoryLimit(QUEUE_MEMORY_LIMIT);
policy.setPendingQueuePolicy(new StorePendingQueueMessageStoragePolicy());
// disable the cache to be sure setBatch is the problem
// will get lots of duplicates
// real problem is sync between cursor and store add - leads to out or order messages
// in the cursor so setBatch can break.
// policy.setUseCache(false);
PolicyMap pMap = new PolicyMap();
pMap.setDefaultEntry(policy);
answer.setDestinationPolicy(pMap);
answer.setDeleteAllMessagesOnStartup(true);
answer.addConnector("tcp://localhost:0");
MemoryUsage memoryUsage = new MemoryUsage();
memoryUsage.setLimit(MEMORY_USAGE);
memoryUsage.setPercentUsageMinDelta(20);
TempUsage tempUsage = new TempUsage();
tempUsage.setLimit(TEMP_USAGE);
StoreUsage storeUsage = new StoreUsage();
storeUsage.setLimit(STORE_USAGE);
SystemUsage systemUsage = new SystemUsage();
systemUsage.setMemoryUsage(memoryUsage);
systemUsage.setTempUsage(tempUsage);
systemUsage.setStoreUsage(storeUsage);
answer.setSystemUsage(systemUsage);
}
/**
* Message listener that is given the Session for transacted consumers
*/
class SessionAwareMessageListener implements MessageListener{
private final List<Message> consumerList;
private final CountDownLatch latch;
private final Session consumerSession;
private Session producerSession;
private MessageProducer producer;
public SessionAwareMessageListener(Session consumerSession, CountDownLatch latch, List<Message> consumerList){
this(null, consumerSession, null, latch, consumerList);
}
public SessionAwareMessageListener(Connection producerConnection, Session consumerSession, String outQueueName,
CountDownLatch latch, List<Message> consumerList){
this.consumerList = consumerList;
this.latch = latch;
this.consumerSession = consumerSession;
if(producerConnection != null){
try {
producerSession = producerConnection.createSession(TRANSACTED, Session.AUTO_ACKNOWLEDGE);
Destination queue = producerSession.createQueue(outQueueName);
producer = producerSession.createProducer(queue);
} catch (JMSException e) {
e.printStackTrace();
}
}
}
@Override
public void onMessage(Message msg) {
try {
if(producer == null){
// sleep to act as a slow consumer
// which will force a mix of direct and polled dispatching
// using the cursor on the broker
Thread.sleep(50);
}else{
producer.send(msg);
if(TRANSACTED) producerSession.commit();
}
} catch (Exception e) {
e.printStackTrace();
}
synchronized(consumerList){
consumerList.add(msg);
if(DEBUG && consumerList.size()%100 == 0) {
int index = consumerList.size()/100;
System.out.print(index-((index/10)*10));
}
if (consumerList.size() == MESSAGE_COUNT) {
latch.countDown();
}
}
if(TRANSACTED){
try {
consumerSession.commit();
} catch (JMSException e) {
e.printStackTrace();
}
}
}
}
}