* JBoss, Home of Professional Open Source

  * Copyright 2005, JBoss Inc., and individual contributors as indicated

  * by the @authors tag. See the copyright.txt in the distribution for a

  * full listing of individual contributors.

  *

  * This is free software; you can redistribute it and/or modify it

  * under the terms of the GNU Lesser General Public License as

  * published by the Free Software Foundation; either version 2.1 of

  * the License, or (at your option) any later version.

  *

  * This software is distributed in the hope that it will be useful,

  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

  * Lesser General Public License for more details.

  *

  * You should have received a copy of the GNU Lesser General Public

  * License along with this software; if not, write to the Free

  * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA

  * 02110-1301 USA, or see the FSF site: http://www.fsf.org.

  */

* The server side representation of a JMS session.

*

* A user must not invoke methods of a session concurrently on different threads, however there are

* situations where multiple threads may access this object concurrently, for instance:

* - A session can be closed when it's connection is closed by the user which might be called on a

*   different thread.

* - A session can be closed when the server determines the connection is dead.

*

* If the session represents a connection consumer's session then the connection consumer will farm

* off messages to different sessions obtained from a pool, these may then cancel/ack etc on

* different threads, but the acks/cancels/etc will end up back here on the connection consumer

* session instance.

*

* @author <a href="mailto:ovidiu@feodorov.com">Ovidiu Feodorov</a>

* @author <a href="mailto:tim.fox@jboss.com">Tim Fox</a>

* @author <a href="mailto:clebert.suconic@jboss.com">Clebert Suconic</a>

* @version <tt>$Revision: 3435 $</tt>

*

* $Id: ServerSessionEndpoint.java 3435 2007-12-06 19:00:46Z timfox $

*/

   // Constants ------------------------------------------------------------------------------------

   private static final Logger log = Logger.getLogger(ServerSessionEndpoint.class);

  

   static final String DUR_SUB_STATE_CONSUMERS = "C";

  

   static final String TEMP_QUEUE_MESSAGECOUNTER_PREFIX = "TempQueue.";

  

   private static final long DELIVERY_WAIT_TIMEOUT = 5 * 1000;

  

   private static final long CLOSE_WAIT_TIMEOUT = 10 * 1000;

     

   // Static ---------------------------------------------------------------------------------------

   // Attributes -----------------------------------------------------------------------------------

   private boolean trace = log.isTraceEnabled();

   private String id;

   private volatile boolean closed;

   private ServerConnectionEndpoint connectionEndpoint;

   private ServerInvokerCallbackHandler callbackHandler;

  

   private ServerPeer sp;

   private Map consumers;

   private Map browsers;

   private PersistenceManager pm;

   private MessageStore ms;

   private DestinationManager dm;

   private IDManager idm;

   private TransactionRepository tr;

   private PostOffice postOffice;

   private int nodeId;

   private int defaultMaxDeliveryAttempts;

   private long defaultRedeliveryDelay;

   private Queue defaultDLQ;

   private Queue defaultExpiryQueue;

   private boolean supportsFailover;

   private boolean replicating;

  

   private Object deliveryLock = new Object();

     

   // Map <deliveryID, Delivery>

   private Map deliveries;

  

   private long deliveryIdSequence;

  

   //Temporary until we have our own NIO transport  

   QueuedExecutor executor = new QueuedExecutor(new LinkedQueue());

  

   private LinkedQueue toDeliver = new LinkedQueue();

  

   private boolean waitingToClose = false;

  

   private Object waitLock = new Object();

  

   private long lastSequence = -1;

  

  

   // Constructors ---------------------------------------------------------------------------------

   ServerSessionEndpoint(String sessionID, ServerConnectionEndpoint connectionEndpoint,

                       boolean replicating) throws Exception

   {

      this.id = sessionID;

      this.connectionEndpoint = connectionEndpoint;

     

      this.replicating = replicating;

     

      callbackHandler = connectionEndpoint.getCallbackHandler();

     

      sp = connectionEndpoint.getServerPeer();

      pm = sp.getPersistenceManagerInstance();

     

      ms = sp.getMessageStore();

     

      dm = sp.getDestinationManager();

     

      postOffice = sp.getPostOfficeInstance();

     

      supportsFailover = connectionEndpoint.getConnectionFactoryEndpoint().isSupportsFailover() && postOffice.isClustered();

     

      idm = sp.getChannelIDManager();

     

      nodeId = sp.getServerPeerID();

     

      tr = sp.getTxRepository();

     

      consumers = new HashMap();

     

    browsers = new HashMap();

     

      defaultDLQ = sp.getDefaultDLQInstance();

     

      defaultExpiryQueue = sp.getDefaultExpiryQueueInstance();

     

      tr = sp.getTxRepository();

     

      defaultMaxDeliveryAttempts = sp.getDefaultMaxDeliveryAttempts();

     

      defaultRedeliveryDelay = sp.getDefaultRedeliveryDelay();

     

      deliveries = new ConcurrentHashMap();

   }

  

   // SessionDelegate implementation ---------------------------------------------------------------

      

   public ConsumerDelegate createConsumerDelegate(JBossDestination jmsDestination,

                                                  String selector,

                                                  boolean noLocal,

                                                  String subscriptionName,

                                                  boolean isCC,

                                                  boolean autoFlowControl) throws JMSException

   {

      try

      {

        //TODO This is a temporary kludge to allow creation of consumers directly on core queues for

        //cluster connections

        //This will disappear once we move all JMS knowledge to the client side

       

        if (jmsDestination.isDirect())

        {

          return createConsumerDelegateDirect(jmsDestination.getName(), selector);

        }

        else

        {               

           return createConsumerDelegateInternal(jmsDestination, selector,

                                                 noLocal, subscriptionName);

        }

      }

      catch (Throwable t)

      {

         throw ExceptionUtil.handleJMSInvocation(t, this + " createConsumerDelegate");

      }

   }

     

  public BrowserDelegate createBrowserDelegate(JBossDestination jmsDestination,

                                                String selector)

      throws JMSException

  {

      try

      {

         return createBrowserDelegateInternal(jmsDestination, selector);

      }

      catch (Throwable t)

      {

         throw ExceptionUtil.handleJMSInvocation(t, this + " createBrowserDelegate");

      }

  }

   public JBossQueue createQueue(String name) throws JMSException

   {

      try

      {

         if (closed)

         {

            throw new IllegalStateException("Session is closed");

         }

        

         ManagedDestination dest = (ManagedDestination)dm.getDestination(name, true);

        

         if (dest == null)

         {

            throw new JMSException("There is no administratively defined queue with name:" + name);

         }       

  

         return new JBossQueue(dest.getName());

      }

      catch (Throwable t)

      {

         throw ExceptionUtil.handleJMSInvocation(t, this + " createQueue");

      }

   }

   public JBossTopic createTopic(String name) throws JMSException

   {

      try

      {

         if (closed)

         {

            throw new IllegalStateException("Session is closed");

         }

        

         ManagedDestination dest = (ManagedDestination)dm.getDestination(name, false);

                 

         if (dest == null)

         {

            throw new JMSException("There is no administratively defined topic with name:" + name);

         }       

  

         return new JBossTopic(name);

      }

      catch (Throwable t)

      {

         throw ExceptionUtil.handleJMSInvocation(t, this + " createTopic");

      }

   }

  

   public void close() throws JMSException

   {

      try

      {

         localClose();

        

         connectionEndpoint.removeSession(id);

      }

      catch (Throwable t)

      {

         throw ExceptionUtil.handleJMSInvocation(t, this + " close");

      }

   }

     

   public long closing(long sequence) throws JMSException

   {

      if (trace) log.trace(this + " closing");

     

      //Need to wait for messages to arrive or they may be lost if connection/session is closed too

      //quickly after sending

     

      if (sequence != -1)

      {

         synchronized (waitLock)

         {          

            long wait = CLOSE_WAIT_TIMEOUT;

           

            while (lastSequence != sequence - 1 && wait > 0)

            {

               long start = System.currentTimeMillis();

               try

               {

                  waitLock.wait(wait);

               }

               catch (InterruptedException e)

               {                

               }

               wait -= (System.currentTimeMillis() - start);

            }

           

            if (wait <= 0)

            {

               log.warn("Timed out waiting for last message");

            }

         }       

      } 

     

      return -1;

   }

   public void send(JBossMessage message, final boolean checkForDuplicates) throws JMSException

   {

      throw new IllegalStateException("Should not be handled here");

   }

  

   public void send(JBossMessage message, final boolean checkForDuplicates, long sequence) throws JMSException

   {

      try

      {               

         connectionEndpoint.sendMessage(message, null, checkForDuplicates);

        

         if (sequence != -1)

         {

            synchronized (waitLock)

            {

               this.lastSequence = sequence;

              

               waitLock.notifyAll();

            }

         }

      }

      catch (Throwable t)

      {

         throw ExceptionUtil.handleJMSInvocation(t, this + " send");

      }

   }

  

   public boolean acknowledgeDelivery(Ack ack) throws JMSException

   {

      try

      {

         return acknowledgeDeliveryInternal(ack);  

      }

      catch (Throwable t)

      {

         throw ExceptionUtil.handleJMSInvocation(t, this + " acknowledgeDelivery");

      }

   }    

        

   public void acknowledgeDeliveries(List acks) throws JMSException

   {   

      if (trace) {log.trace(this + " acknowledges deliveries " + acks); }

     

      try

      {

         Iterator iter = acks.iterator();

        

         while (iter.hasNext())

         {

            Ack ack = (Ack)iter.next();

           

            acknowledgeDeliveryInternal(ack);

         }

      }

      catch (Throwable t)

      {

         throw ExceptionUtil.handleJMSInvocation(t, this + " acknowledgeDeliveries");

      }

   }

            

   public void cancelDelivery(Cancel cancel) throws JMSException

   {

      if (trace) {log.trace(this + " cancelDelivery " + cancel); }

     

      try

      {

         Delivery del = cancelDeliveryInternal(cancel);

        

         if (del != null)

         {        

           //Prompt delivery

           promptDelivery((Channel)del.getObserver());

         }

      }

      catch (Throwable t)

      {

         throw ExceptionUtil.handleJMSInvocation(t, this + " cancelDelivery");

      }    

   }           

   public void cancelDeliveries(List cancels) throws JMSException

   {

      if (trace) {log.trace(this + " cancels deliveries " + cancels); }

       

      try

      {

         // deliveries must be cancelled in reverse order

         Set channels = new HashSet();

                         

         for (int i = cancels.size() - 1; i >= 0; i--)

         {

            Cancel cancel = (Cancel)cancels.get(i);      

           

            if (trace) { log.trace(this + " cancelling delivery " + cancel.getDeliveryId()); }

                       

            Delivery del = cancelDeliveryInternal(cancel);

           

            if (del != null)

            {             

              channels.add(del.getObserver());

            }

         }

                

         if (trace) { log.trace("Cancelled deliveries"); }

        

         // need to prompt delivery for all affected channels

        

         promptDelivery(channels);

      }

      catch (Throwable t)

      {

         throw ExceptionUtil.handleJMSInvocation(t, this + " cancelDeliveries");

      }

   }        

  

   public void recoverDeliveries(List deliveryRecoveryInfos, String oldSessionID) throws JMSException

   {

      if (trace) { log.trace(this + " recovers deliveries " + deliveryRecoveryInfos); }

      try

      {

         if (!postOffice.isClustered())

         {

            throw new IllegalStateException("Recovering deliveries but post office is not clustered!");

         }

        

         long maxDeliveryId = 0;

                 

         //Sort into different list for each channel

         Map ackMap = new HashMap();

                 

         for (Iterator iter = deliveryRecoveryInfos.iterator(); iter.hasNext(); )

         {

            DeliveryRecovery deliveryInfo = (DeliveryRecovery)iter.next();

               

            String queueName = deliveryInfo.getQueueName();

            List acks = (List)ackMap.get(queueName);

           

            if (acks == null)

            {

               acks = new ArrayList();

              

               ackMap.put(queueName, acks);

            }

           

            acks.add(deliveryInfo);

         } 

         Iterator iter = ackMap.entrySet().iterator();

        

         while (iter.hasNext())

         {

            Map.Entry entry = (Map.Entry)iter.next();

           

            String queueName = (String)entry.getKey();

           

            //Look up the queue

            Binding binding = postOffice.getBindingForQueueName(queueName);

           

            Queue queue = binding.queue;

           

            if (queue == null)

            {

               throw new IllegalStateException("Cannot find queue with queue name: " + queueName);

            }

           

            List acks = (List)entry.getValue();

           

            List ids = new ArrayList(acks.size());

           

            for (Iterator iter2 = acks.iterator(); iter2.hasNext(); )

            {

               DeliveryRecovery info = (DeliveryRecovery)iter2.next();

              

               ids.add(new Long(info.getMessageID()));

            }

           

            JMSCondition cond = (JMSCondition)binding.condition;

           

            ManagedDestination dest =

               sp.getDestinationManager().getDestination(cond.getName(), cond.isQueue());

           

            if (dest == null)

            {

               throw new IllegalStateException("Cannot find managed destination with name " +

                  cond.getName() + " isQueue" + cond.isQueue());

            }

           

            Queue dlqToUse =

               dest.getDLQ() == null ? defaultDLQ : dest.getDLQ();

           

            Queue expiryQueueToUse =

               dest.getExpiryQueue() == null ? defaultExpiryQueue : dest.getExpiryQueue();

           

            int maxDeliveryAttemptsToUse =

               dest.getMaxDeliveryAttempts() == -1 ? defaultMaxDeliveryAttempts : dest.getMaxDeliveryAttempts();

            List dels = queue.recoverDeliveries(ids);

            Iterator iter2 = dels.iterator();

           

            Iterator iter3 = acks.iterator();

           

            while (iter2.hasNext())

            {

               Delivery del = (Delivery)iter2.next();

              

               DeliveryRecovery info = (DeliveryRecovery)iter3.next();

              

               long deliveryId = info.getDeliveryID();

              

               maxDeliveryId = Math.max(maxDeliveryId, deliveryId);

              

               if (trace) { log.trace(this + " Recovered delivery " + deliveryId + ", " + del); }

              

               deliveries.put(new Long(deliveryId),

                              new DeliveryRecord(del, dlqToUse, expiryQueueToUse, dest.getRedeliveryDelay(),

                                  maxDeliveryAttemptsToUse, queueName, supportsFailover, deliveryId));

              

               //We want to replicate the deliveries to the new backup, but we don't want a response since that would cause actual delivery

               //to occur, which we don't want since the client already has the deliveries

              

               if (supportsFailover)

               {

                 postOffice.sendReplicateDeliveryMessage(queueName, id, del.getReference().getMessage().getMessageID(), deliveryId, false, true);

               }

            }

         }

        

         iter = postOffice.getAllBindings().iterator();

        

         while (iter.hasNext())

         {

           Binding binding = (Binding)iter.next();

          

           if (binding.queue.isClustered() && binding.queue.isRecoverable())

           {

             // Remove any stranded refs corresponding to refs that might have been in the client buffer but not consumed

             binding.queue.removeStrandedReferences(oldSessionID);

           }

         }

        

         deliveryIdSequence = maxDeliveryId + 1;

      }

      catch (Throwable t)

      {

         throw ExceptionUtil.handleJMSInvocation(t, this + " recoverDeliveries");

      }

   }

  

   public void addTemporaryDestination(JBossDestination dest) throws JMSException

   {

      try

      {

         if (closed)

         {

            throw new IllegalStateException("Session is closed");

         }

         if (!dest.isTemporary())

         {

            throw new InvalidDestinationException("Destination:" + dest +

               " is not a temporary destination");

         }

         connectionEndpoint.addTemporaryDestination(dest);

        

         // Register with the destination manager

        

         ManagedDestination mDest;

        

         int fullSize = connectionEndpoint.getDefaultTempQueueFullSize();

         int pageSize = connectionEndpoint.getDefaultTempQueuePageSize();

         int downCacheSize = connectionEndpoint.getDefaultTempQueueDownCacheSize();

        

         //Temporary destinations are clustered if the post office is clustered

         if (dest.isTopic())

         {

            mDest = new ManagedTopic(dest.getName(), fullSize, pageSize, downCacheSize, postOffice.isClustered());

         }

         else

         {

            mDest = new ManagedQueue(dest.getName(), fullSize, pageSize, downCacheSize, postOffice.isClustered());

         }

        

         mDest.setTemporary(true);

        

         dm.registerDestination(mDest);

        

         if (dest.isQueue())

         {           

            Queue coreQueue = new MessagingQueue(nodeId, dest.getName(),

                                     idm.getID(), ms, pm, false, -1, null,

                                       fullSize, pageSize, downCacheSize, postOffice.isClustered(),

                                       sp.getRecoverDeliveriesTimeout());

       

            Condition cond = new JMSCondition(true, dest.getName());

           

           // make a binding for this temporary queue

           

            // temporary queues need to bound on ALL nodes of the cluster

            postOffice.addBinding(new Binding(cond, coreQueue, true), postOffice.isClustered());  

           

            coreQueue.activate();

         }        

      }

      catch (Throwable t)

      {

         throw ExceptionUtil.handleJMSInvocation(t, this + " addTemporaryDestination");

      }

   }

  

   public void deleteTemporaryDestination(JBossDestination dest) throws JMSException

   {

      try

      {

         if (closed)

         {

            throw new IllegalStateException("Session is closed");

         }

  

         if (!dest.isTemporary())

         {

            throw new InvalidDestinationException("Destination:" + dest +

                                                  " is not a temporary destination");

         }

        

         ManagedDestination mDest = dm.getDestination(dest.getName(), dest.isQueue());

        

         if (mDest == null)

         {

            throw new InvalidDestinationException("No such destination: " + dest);

         }

                 

         if (dest.isQueue())

         {

           Binding binding = postOffice.getBindingForQueueName(dest.getName());

          

           if (binding == null)

           {

             throw new IllegalStateException("Cannot find binding for queue " + dest.getName());

           }

          

           if (binding.queue.getLocalDistributor().getNumberOfReceivers() != 0)

           {

             throw new IllegalStateException("Cannot delete temporary queue if it has consumer(s)");

           }

          

           // temporary queues must be unbound on ALL nodes of the cluster

          

            postOffice.removeBinding(dest.getName(), postOffice.isClustered());                  

         }

         else

         {

            //Topic           

            Collection queues = postOffice.getQueuesForCondition(new JMSCondition(false, dest.getName()), true);          

                          

            if (!queues.isEmpty())

           {

              throw new IllegalStateException("Cannot delete temporary topic if it has consumer(s)");

           }

                       

            // There is no need to explicitly unbind the subscriptions for the temp topic, this is because we

            // will not get here unless there are no bindings.

            // Note that you cannot create surable subs on a temp topic

         }

        

         connectionEndpoint.removeTemporaryDestination(dest);        

        

         dm.unregisterDestination(mDest);                            

      }

      catch (Throwable t)

      {

         throw ExceptionUtil.handleJMSInvocation(t, this + " deleteTemporaryDestination");

      }

   }

  

   public void unsubscribe(String subscriptionName) throws JMSException

   {

      log.trace(this + " unsubscribing " + subscriptionName);

     

      try

      {

         if (closed)

         {

            throw new IllegalStateException("Session is closed");

         }

         if (subscriptionName == null)

         {

            throw new InvalidDestinationException("Destination is null");

         }

  

         String clientID = connectionEndpoint.getClientID();

  

         if (clientID == null)

         {

            throw new JMSException("null clientID on connection");

         }

        

         String queueName = MessageQueueNameHelper.createSubscriptionName(clientID, subscriptionName);

        

         Binding binding = postOffice.getBindingForQueueName(queueName);

        

         if (binding == null)

         {

            throw new InvalidDestinationException("Cannot find durable subscription with name " +

                                                  subscriptionName + " to unsubscribe");

         }

        

         Queue sub = binding.queue;        

        

         // Section 6.11. JMS 1.1.

         // "It is erroneous for a client to delete a durable subscription while it has an active

         // TopicSubscriber for it or while a message received by it is part of a current

         // transaction or has not been acknowledged in the session."

        

         if (sub.getLocalDistributor().getNumberOfReceivers() != 0)

         {

            throw new IllegalStateException("Cannot unsubscribe durable subscription " +

                                            subscriptionName + " since it has active subscribers");

         }

        

         //Also if it is clustered we must disallow unsubscribing if it has active consumers on other nodes

        

         if (sub.isClustered() && postOffice.isClustered())

         {

           Replicator rep = (Replicator)postOffice;

          

           Map map = rep.get(sub.getName());

          

           if (!map.isEmpty())

           {

             throw new IllegalStateException("Cannot unsubscribe durable subscription " +

                     subscriptionName + " since it has active subscribers on other nodes");

           }

         }

        

         postOffice.removeBinding(sub.getName(), sub.isClustered() && postOffice.isClustered());        

        

         String counterName = TopicService.SUBSCRIPTION_MESSAGECOUNTER_PREFIX + sub.getName();

        

         MessageCounter counter = sp.getMessageCounterManager().unregisterMessageCounter(counterName);

        

         if (counter == null)

         {

            throw new IllegalStateException("Cannot find counter to remove " + counterName);

         }

      }

      catch (Throwable t)

      {

         throw ExceptionUtil.handleJMSInvocation(t, this + " unsubscribe");

      }

   }

  

   // Public ---------------------------------------------------------------------------------------

  

   public ServerConnectionEndpoint getConnectionEndpoint()

   {

      return connectionEndpoint;

   }

   public String toString()

   {

      return "SessionEndpoint[" + id + "]";

   }

  

   public void deliverAnyWaitingDeliveries(String queueName) throws Exception

   {

     //  First deliver any waiting deliveries

    

     if (trace) { log.trace("Delivering any waiting deliveries: " + queueName); }

    

     List toAddBack = null;

    

     while (true)

     {

       DeliveryRecord dr = (DeliveryRecord)toDeliver.poll(0);

      

       if (dr == null)

       {

         break;

       }

      

       if (trace) { log.trace("Considering " + dr); }

      

       if (queueName == null || dr.queueName.equals(queueName))

       {      

         //Need to synchronized to prevent the delivery being performed twice

         synchronized (dr)

         {          

           performDelivery(dr.del.getReference(), dr.deliveryID, dr.getConsumer());

         

           dr.waitingForResponse = false;

         }

       }

       else

       {

         if (toAddBack == null)

         {

           toAddBack = new ArrayList();

         }

        

         toAddBack.add(dr);

       }

     }

    

     if (toAddBack != null)

     {

       Iterator iter = toAddBack.iterator();

      

       while (iter.hasNext())

       {

         toDeliver.put(iter.next());

       }

     }

    

     if (trace) { log.trace("Done delivering"); }

   }

  

   public boolean collectDeliveries(Map map, boolean firstNode, String queueName) throws Exception

   {

     if (trace) { log.trace("Collecting deliveries"); }

    

     boolean gotSome = false;

         

     if (!firstNode && replicating)

     {      

       if (trace) { log.trace("Now collecting"); }

           

       Iterator iter = deliveries.entrySet().iterator();

      

       while (iter.hasNext())

       {

         Map.Entry entry = (Map.Entry)iter.next();

        

         Long l = (Long)entry.getKey();

        

         long deliveryID = l.longValue();

        

         DeliveryRecord rec = (DeliveryRecord)entry.getValue();

        

         if (rec.replicating && (queueName == null || rec.queueName.equals(queueName)))

         {

           Map ids = (Map)map.get(rec.queueName);

          

           if (ids == null)

           {

             ids = new HashMap();

            

             map.put(rec.queueName, ids);

           }

          

           ids.put(new Long(rec.del.getReference().getMessage().getMessageID()), id);

          

           gotSome = true;

          

           boolean notify = false;

          

           //Need to synchronize to prevent delivery occurring more than once - e.g.

           //if replicateDeliveryResponseReceived occurs curently with this

           synchronized (rec)

           {

             if (rec.waitingForResponse)

             {

               //Do the delivery now

              

               performDelivery(rec.del.getReference(), deliveryID, rec.getConsumer());         

               

                rec.waitingForResponse = false;

               

                notify = true;

             }

           }

             

           if (notify)

           {

              synchronized (deliveryLock)

              {

                if (waitingToClose)

                {

                  deliveryLock.notifyAll();

                }

              }                     

           }

         }

       }

     }

    

     if (trace) { log.trace("Collected " + map.size() + " deliveries"); }

    

     return gotSome;

   }

     

   public void replicateDeliveryResponseReceived(long deliveryID) throws Exception

   {

     //We look up the delivery in the list and actually perform the delivery

    

     if (trace) { log.trace(this + " replicate delivery response received for delivery " + deliveryID); }

    

     DeliveryRecord rec = (DeliveryRecord)deliveries.get(new Long(deliveryID));

    

     if (rec == null)

     {

       //This is ok - may happen at failover

        

        // This can happen when the failover node is being changed

         // E.g. failover node changes, replicates start getting sent to the new failover node,

         // then the new node requests to collect the deliveries from this node, at which point we deliver

         // all waiting deliveries. Then the responses to the original ones come back.

         // So we can ignore them

        

         return;

     }

         

     boolean delivered = false;    

    

    while (true)

     {

       DeliveryRecord dr = (DeliveryRecord)toDeliver.peek();

                        

       if (dr == null)

       {

         if (trace) { log.trace("No more deliveries in list"); }

        

         break;