Package com.sun.ejb.ee.sfsb.store

Source Code of com.sun.ejb.ee.sfsb.store.ReplicatedSFSBStoreManager$DispatchThread

/*
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
*
* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
*
* The contents of this file are subject to the terms of either the GNU
* General Public License Version 2 only ("GPL") or the Common Development
* and Distribution License("CDDL") (collectively, the "License").  You
* may not use this file except in compliance with the License. You can obtain
* a copy of the License at https://glassfish.dev.java.net/public/CDDL+GPL.html
* or glassfish/bootstrap/legal/LICENSE.txt.  See the License for the specific
* language governing permissions and limitations under the License.
*
* When distributing the software, include this License Header Notice in each
* file and include the License file at glassfish/bootstrap/legal/LICENSE.txt.
* Sun designates this particular file as subject to the "Classpath" exception
* as provided by Sun in the GPL Version 2 section of the License file that
* accompanied this code.  If applicable, add the following below the License
* Header, with the fields enclosed by brackets [] replaced by your own
* identifying information: "Portions Copyrighted [year]
* [name of copyright owner]"
*
* Contributor(s):
*
* If you wish your version of this file to be governed by only the CDDL or
* only the GPL Version 2, indicate your decision by adding "[Contributor]
* elects to include this software in this distribution under the [CDDL or GPL
* Version 2] license."  If you don't indicate a single choice of license, a
* recipient has the option to distribute your version of this file under
* either the CDDL, the GPL Version 2 or to extend the choice of license to
* its licensees as provided above.  However, if you add GPL Version 2 code
* and therefore, elected the GPL Version 2 license, then the option applies
* only if the new code is made subject to such option by the copyright
* holder.
*/
/*
* ReplicatedSFSBStoreManager.java
*
* Created on June 1, 2006, 10:58 AM
*
*/

package com.sun.ejb.ee.sfsb.store;

import com.sun.appserv.ha.spi.BackingStore;
import com.sun.appserv.ha.spi.BackingStoreException;
import com.sun.appserv.ha.spi.BackingStoreFactory;
import com.sun.appserv.ha.spi.BackingStoreRegistry;
import com.sun.appserv.ha.util.SimpleMetadata;
import com.sun.appserv.util.cache.BaseCache;
import com.sun.ejb.spi.sfsb.store.SFSBBeanState;
import com.sun.ejb.spi.sfsb.store.SFSBStoreManagerException;
import com.sun.ejb.spi.sfsb.store.SFSBTxStoreManager;
import com.sun.ejb.spi.stats.MonitorableSFSBStoreManager;
import com.sun.enterprise.ee.web.sessmgmt.*;
import com.sun.enterprise.web.ServerConfigLookup;
import com.sun.logging.LogDomains;
import org.apache.catalina.Globals;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.lang.reflect.InvocationTargetException;
import java.security.AccessController;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;


/**
*
* @author Larry White
*/
public class ReplicatedSFSBStoreManager extends BaseSFSBStoreManager
    implements SFSBTxStoreManager, MonitorableSFSBStoreManager, ReplicationManager {
   
    final static String MODE_EJB
        = ReplicationState.MODE_EJB;

    protected static java.util.concurrent.atomic.AtomicLong unique =
        new AtomicLong(0);   
   
    public final static String LOGGER_MEM_REP
        = "com.sun.ejb.ee.sfsb.store";
   
    private final static Level TRACE_LEVEL = Level.FINE;
   
    final static String SESSION_MANAGER_PROPERTY =
            ReplicationState.SESSION_MANAGER_PROPERTY; // used during jxtabackingstore creation to store a reference to this session manager in jxtabackingstore.
    final static String DUPLICATE_IDS_SEMANTICS_PROPERTY
        = ReplicationState.DUPLICATE_IDS_SEMANTICS_PROPERTY;
   
    private final static String MESSAGE_BROADCAST_LOAD_RECEIVED
        = ReplicationState.MESSAGE_BROADCAST_LOAD_RECEIVED;   
    public final static String MESSAGE_BROADCAST_EJB_ID_QUERY
        = "broadcastfindejbids";
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_RESPONSE
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_RESPONSE;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_RESPONSE
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_RESPONSE;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_RECONCILIATION_COMPLETE_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_RECONCILIATION_COMPLETE_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_RESPONSE
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_RESPONSE;
   
    /** Pool of ReplicatedEjbStore StorePool elements
     */
    StorePool _pool = null;
   
    /**
     * The singleton store.
     */   
    protected ReplicatedEjbStore _store = null;   
   
    protected static int _maxBaseCacheSize = 4096;
    protected static float _loadFactor = 0.75f;
    protected static final int _concurrencyLevel = 100;

    ConcurrentHashMap stillOwnedEjbIdsForActiveCacheReconciliation = null;

    protected AtomicBoolean _activeCacheReconciliationOngoing
        = new AtomicBoolean(false);
   
    /** Logger for logging
     */
    //private final static Logger _logger = LogDomains.getLogger(LogDomains.EJB_LOGGER);   


    private static final Logger _logger
        = Logger.getLogger(LOGGER_MEM_REP);
    private static final Logger _salogger = ReplicationUtil.getSALogger();   

    /**
     * the list of method names that are broadcasts or unicast
     */
    private static List broadcastMethods
        = Arrays.asList(            
            MESSAGE_BROADCAST_EJB_ID_QUERY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_ADVISORY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_ADVISORY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_RESPONSE,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_ADVISORY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_RESPONSE,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_ADVISORY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_RESPONSE,           
            MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_RECONCILIATION_COMPLETE_ADVISORY);

    /**
     * the list of method names that are removes
     */
    private static List removeMethods
        = Arrays.asList(ReplicationState.REMOVE_COMMAND,
            ReplicationState.REMOVE_SYNCHRONIZED_COMMAND);

    private static List saveMethods
  = Arrays.asList(ReplicationState.SAVE_COMMAND);
   
    static
  {
            checkSessionCacheProperties();
            registerBroadcastMethods();
            registerRemoveMethods();
      registerSaveMethods();
  }    

    protected static boolean checkSessionCacheProperties() {
        boolean result = false;
  try
        {
            Properties props = System.getProperties();
            String cacheSize=props.getProperty("EJB_SESSION_CACHE_MAX_BASE_CACHE_SIZE");
            if(null!=cacheSize) {
                _maxBaseCacheSize = (new Integer (cacheSize).intValue());
           
            String loadFactor=props.getProperty("EJB_SESSION_CACHE_MAX_BASE_LOAD_FACTOR");
            if(null!=loadFactor) {
                _loadFactor = (new Float (loadFactor).floatValue());
            }
//            if (_logger.isLoggable(Level.FINE)) {
//                _logger.fine("_maxBaseCacheSize=" + _maxBaseCacheSize +
//                             " _loadFactor=" + _loadFactor);
//            }
        } catch(Exception e)
        {
            //do nothing accept defaults
        }
        return result;
    }
   
    private static ReplicationMessageRouter getRouter() {
        ReplicationMessageRouter receiver = null;
        if (Globals.IS_SECURITY_ENABLED) {
            receiver = (ReplicationMessageRouter)
                AccessController.doPrivileged(
                    new PrivilegedGetReplicationMessageRouter());
        } else {
            receiver = ReplicationMessageRouter.createInstance();
        }
        return receiver;
    }

    protected static void registerBroadcastMethods() {
        ReplicationMessageRouter router = getRouter();
        router.registerBroadcastMethodList(broadcastMethods);
    }

    protected static void registerRemoveMethods() {
        ReplicationMessageRouter router = getRouter();
        router.registerRemoveMethodList(removeMethods);
    }

    protected static void registerSaveMethods() {
  ReplicationMessageRouter router = getRouter();
  router.registerSaveMethodList(saveMethods);
    }
   
    public String getPassedInPersistenceType() {
        return _passedInPersistenceType;
    }   
   
    public void setPassedInPersistenceType(String persistenceType) {
        _passedInPersistenceType = persistenceType;
    }

    ReplicaCache getReplicaCache() {
        return replicaCache;
    }

    public void setReplicaCache(ReplicaCache replicaCache) {
        this.replicaCache = replicaCache;
    }   

    public boolean isActiveCacheReconciliationOngoing() {
        return _activeCacheReconciliationOngoing.get();
    }

    public void setActiveCacheReconciliationOngoing(boolean value) {
        _activeCacheReconciliationOngoing.set(value);
    }

    protected String _rollingUpgradeBackupDirectory = null;
    protected boolean _skipRollingUpgradeBackupRestore = false;

    /**
     * get the rollingUpgradeBackupDirectory
     */
    public String getRollingUpgradeBackupDirectory() {
        return _rollingUpgradeBackupDirectory;
    }

    /**
     * set the rollingUpgradeBackupDirectory
     * @param value
     */
    public void setRollingUpgradeBackupDirectory(String value) {
        _rollingUpgradeBackupDirectory = value;
    }

    public <V> V __load(String id, String version, JxtaBackingStoreImpl jxtaBackingStore)
            throws BackingStoreException {
        ReplicatedEjbStore store = getSingletonStore();
        return (V)store.__load(id);
    }

    boolean replicationCompressionEnabled = false;

    boolean isReplicationCompressionEnabled() {
        return replicationCompressionEnabled;
    }

    /** should skip backup and restore step for rolling upgrade */
    boolean isSkipRollingUpgradeBackupRestore() {
        return _skipRollingUpgradeBackupRestore;
    }

    /**
     * set the skipRollingUpgradeBackupRestore
     * @param value
     */
    public void setSkipRollingUpgradeBackupRestore(boolean value) {
        _skipRollingUpgradeBackupRestore = value;
    }

    protected ConcurrentHashMap<String, ReplicationState> ownedReplicaListsReceivedForActiveCacheRestoration = null;
    protected ConcurrentHashMap<String, ReplicationState> ownedIdsForReplicaCacheReconciliation = null;
    protected ConcurrentHashMap<String, ReplicationState> ownedIdsForActiveCacheReconciliation = null;

    private volatile CountDownLatch reconcileReplicaCacheDoneSignal = null;
    private volatile CountDownLatch restoreActiveCacheDoneSignal = null;
    private volatile CountDownLatch getIdsForReplicaCacheReconciliationDoneSignal = null;
    private volatile CountDownLatch getIdsForActiveCacheReconciliationDoneSignal = null;

    /**
    * the passed in persistence type may be replicated or extension type
    */   
    protected String _passedInPersistenceType = null;    
   
    /**
    * Our Replicator instance
    */
    protected BackingStore backingStore = null;

    /**
    * Our cleanup thread
    */   
    protected CleanupThread cleanupThread = null;
   
    /**
     *   return the containerId as a String
     *  needed to implement ReplicationManager 
     */   
    public String getApplicationId() {
        return getContainerID();
    }   
   
    /**
    * get the backingStore
    */
    public BackingStore getBackingStore() {
        if(backingStore == null) {
            this.createBackingStore();
        }
        return backingStore;
    }
   
    /**
    * set the backing store
    * @param aBackingStore
    */
    public void setBackingStore(BackingStore aBackingStore) {
        backingStore = aBackingStore;
    }

    boolean isThirdPartyBackingStoreInUse() {
        BackingStore backingStore
            = getBackingStore();
        return (!(backingStore instanceof JxtaBackingStoreImpl));
    }

    /**
    * create the backingStore
    */    
    void createBackingStore() {
        //BackingStoreFactory storeFactory = new JxtaBackingStoreFactory();
        BackingStoreFactory storeFactory = getBackingStoreFactory();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("storeFactory: " + storeFactory);
        }        
        BackingStoreRegistry backingStoreRegistry
            = BackingStoreRegistry.getInstance();
        Properties inputEnv
            = backingStoreRegistry.getFactoryClassEnv(getPassedInPersistenceType());
        Properties env = (Properties)inputEnv.clone();       
        //this is always true for sfsbs
        if(env != null) {
            env.put(DUPLICATE_IDS_SEMANTICS_PROPERTY, true);
            env.put(SESSION_MANAGER_PROPERTY, this);
        }
        BackingStore backingStore = null;
        try {
            backingStore = storeFactory.createBackingStore(
                        this.getContainerID(), //appid
                        String.class,
                        SimpleMetadata.class,     //type
                        env);
        } catch (BackingStoreException ex) {
            //deliberate no-op
        }       
        if(backingStore != null) {
            if(backingStore instanceof JxtaBackingStoreImpl) {
                ((JxtaBackingStoreImpl)backingStore).setMode(MODE_EJB);
            }
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("backingStore: " + backingStore);
        }       
        this.setBackingStore(backingStore);
    }

    /**
    * create the backingStore factory based on the passed
    * in persistence-type
    */   
    protected BackingStoreFactory getBackingStoreFactory() {
        BackingStoreFactory backingStoreFactory = new JxtaBackingStoreFactory();
        BackingStoreRegistry backingStoreRegistry
            = BackingStoreRegistry.getInstance();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("getBackingStoreFactory:passedInPersistenceType=" + getPassedInPersistenceType());
        }       
        if(getPassedInPersistenceType() == null) {
            return backingStoreFactory;
        }
        String factoryClassName
            = backingStoreRegistry.getFactoryClassName(this.getPassedInPersistenceType());
        return getBackingStoreFactoryFromName(factoryClassName);
    }
   
    private BackingStoreFactory getBackingStoreFactoryFromName(String className) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("getBackingStoreFactoryFromName:className: " + className);
        }          
        BackingStoreFactory backingStoreFactory = new JxtaBackingStoreFactory();
        try {
            backingStoreFactory =
                (BackingStoreFactory) (Class.forName(className)).newInstance();
        } catch (Exception ex) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("ReplicatedSFSBStoreManager:unable to create backing store factory");
            }           
        }
        return backingStoreFactory;
    }            
   
    /** Creates a new instance of ReplicatedSFSBStoreManager */
    public ReplicatedSFSBStoreManager() {
        super();
        /*
        if (_logger == null) {
            _logger = LogDomains.getLogger(LogDomains.EJB_LOGGER);
        }
         */
        if(_logger.isLoggable(Level.FINEST)) {
            _logger.log(Level.FINEST, "ReplicatedSFSBStoreManager  loaded successfully......");
        }
        StoreFactory haStoreFactory = new ReplicatedEjbStoreFactory();
        _pool = new StorePool(StorePool.DEFAULT_INITIAL_SIZE,
        StorePool.DEFAULT_UPPER_SIZE,
        StorePool.DEFAULT_POLL_TIME, haStoreFactory);
        this.setSingletonStore((ReplicatedEjbStore)haStoreFactory.createHAStore());

        //initialize locally passivated sessions cache
        locallyPassivatedSessions = new BaseCache();
        locallyPassivatedSessions.init(_maxBaseCacheSize, _loadFactor, null);
        replicatedSessionMonitors = new ReplicationSessionMonitors(_logger, _maxBaseCacheSize, _loadFactor);
        ServerConfigLookup lookup = new ServerConfigLookup();
        replicationCompressionEnabled = lookup.isReplicationCompression();
        cleanupThread = new CleanupThread();
   

    /**
    * get the server instance name
    */   
    public String getInstanceName() {
        return ReplicationUtil.getInstanceName();
    }

    /**
    * Our ReplicaCache - the receiving side for replicas
    */
    protected ReplicaCache replicaCache = null;
   
    /**
    * Our cache of locally passivated SFSBBeanState objects
    * keyed by id
    */
    protected BaseCache locallyPassivatedSessions = new BaseCache();
   
    /**
    * Our cache of monitor objects
    * keyed by id
    */
    protected ReplicationSessionMonitors replicatedSessionMonitors = null;

    /**
    * get the replicated sessions cache
    */
    public BaseCache getReplicatedSessions() {
        return replicaCache.getReplicatedSessions();
    }
   
    public Object getReplicationSessionMonitor(String id) {
        return replicatedSessionMonitors.get(id);
    }   
   
    public int processExpiredSessions() {
        backgroundProcess();
        return 0;
    }

    private void backgroundProcess() {
        replicatedSessionMonitors.processExpired();
        replicaCache.processExpiredReplicas();
    }
   
    /**
    * put the replication state into the replica cache
    * @param sessionState
    */
    protected void putInReplicationCache(ReplicationState sessionState) {
        replicaCache.putInReplicationCache(sessionState);
    }
   
    /**
     * get the replication State from the replica cache based on the id
     * @param id
     */   
    protected ReplicationState getFromReplicationCache(String id) {
        return replicaCache.getFromReplicationCache(id);
    }

    /**
    * remove the replication state from the replica cache
    * @param sessionState
    */   
    protected void removeFromReplicationCache(ReplicationState sessionState) {
        if(sessionState == null) {
            return;
        }
        removeFromReplicationCache((String)sessionState.getId());
    }
   
    /**
     * remove the replication state from replica cache based on
     * the id of replica state
     * @param id id of session replica state
     */    
    protected ReplicationState removeFromReplicationCache(String id) {
        return replicaCache.removeFromReplicationCache(id);
    }
   
    /**
    * return and remove the replication state from the replica cache
    * based on the id
    * @param id
    */    
    protected ReplicationState transferFromReplicationCache(String id) {
        return replicaCache.transferFromReplicationCache(id);
    }   
   
    //*****************************************************
   
    /**
    * get the locally passivated sessions cache
    */
    public BaseCache getLocallyPassivatedSessions() {
        return locallyPassivatedSessions;
    }
 
    /**
    * set the locally passivated sessions cache
    * @param sesstable
    */
    public void setLocallyPassivatedSessions(BaseCache sesstable) {
        locallyPassivatedSessions = sesstable;
    }

    /**
    * put the beanState in the locallyPassivatedSessions cache
    * @param beanState
    */    
    protected void putInLocallyPassivatedCache(SFSBBeanState beanState) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicatedSFSBStoreManager>>putInLocallyPassivatedCache id: " + beanState.getContainerId());
        }       
        if(locallyPassivatedSessions == null) {
            return;
        }
        locallyPassivatedSessions.put(beanState.getId(), beanState);
    }

    /**
    * get the beanState in the locallyPassivatedSessions cache
    * based on the id
    * @param id
    */    
    protected SFSBBeanState getFromLocallyPassivatedCache(String id) {   
        return (SFSBBeanState)locallyPassivatedSessions.get(id);
    }

    /**
    * remove the beanState from the locallyPassivatedSessions cache
    * @param beanState
    */   
    protected void removeFromLocallyPassivatedCache(SFSBBeanState beanState) {
        if(locallyPassivatedSessions == null) {
            return;
        }
        locallyPassivatedSessions.remove(beanState.getId());
    }
   
    /**
    * remove the beanState with id from the locallyPassivatedSessions cache
    * @param id
    */   
    protected void removeIdFromLocallyPassivatedCache(Object id) {
        if(locallyPassivatedSessions == null) {
            return;
        }
        locallyPassivatedSessions.remove(id);
    }   
   
    /** return the StorePool this manager holds
     * @return Returns the pool it holds
     */
    public StorePool getStorePool() {
        return _pool;
    }
   
    /** set the StorePool
     * @param pool pool of ReplicatedEjbStore
     */
    public void setStorePool(StorePool pool) {
        _pool = pool;
    }
   
    // singleton store was set during initialization of this manager
    public void setSingletonStore(ReplicatedEjbStore store) {
        _store = store;
        _store.setContainer(this.getContainer());
        _store.setClusterID(this.getClusterID());
        _store.setContainerId(this.getContainerID());
        _store.setSFSBStoreManager(this);       
    }

    public ReplicatedEjbStore getSingletonStore() {
        return _store;
    }

    /** Returns a store from the pool This method intializes the store with right parameters
     * @return returns ReplicatedEjbStore
     */
    private ReplicatedEjbStore getStore() {
        return getStore(ReplicationUtil.isSynchronousReplicationConfigured());
    }
   
    /** Returns a store from the pool This method intializes the store with right parameters
     * @return returns ReplicatedEjbStore
     */
    private ReplicatedEjbStore getStore(boolean useSingleton) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicatedSFSBStoreManager", "getStore");
        }
        if(useSingleton) {
            return this.getSingletonStore();
        }        
        ReplicatedEjbStore store = null;
        try {
            store = (ReplicatedEjbStore) _pool.take();
            store.setContainer(this.getContainer());
            store.setClusterID(this.getClusterID());
            store.setContainerId(this.getContainerID());
            store.setSFSBStoreManager(this);
            if(_logger.isLoggable(Level.FINEST)) {
                _logger.log(Level.FINEST,
                    "ReplicatedSFSBStoreManager.getStore returning   " + store);
            }
            return store;
        }
        catch (Exception e) {
            // FIXME evaluate log level
            if (_logger.isLoggable(Level.FINE)) {
                _logger.log(Level.FINE, "exception occurred in getStore", e);
            }
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicatedSFSBStoreManager", "getStore", store);
        }
        return store;
    }

    /**
     *  Returns (puts) a store back to the pool
     */
    private void putStore(ReplicatedEjbStore store) {
        putStore(store, ReplicationUtil.isSynchronousReplicationConfigured());
    }
   
    /**
     *  Returns (puts) a store back to the pool
     */
    private void putStore(ReplicatedEjbStore store, boolean useSingleton) {
        if(useSingleton) {
            return;
        }
        if (store != null) {
            ( (ReplicatedEjbStore) store).setContainer(null);       
            try {
                StorePool storePool = this.getStorePool();
                if(storePool != null) {
                    storePool.put( (StorePoolElement) store);
                }
            }
            catch (InterruptedException ex1) {
                // FIXME evaluate log level
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.log(Level.FINE, "exception occurred in putStore", ex1);
                }
            }
        }
    }
   
    // SFSBStoreManager methods
   
    /** Saves the state of the bean
     * @param beanState SFSBBeanState
     * @throws SFSBStoreManagerException
     */
    public void checkpointSave(SFSBBeanState beanState)
        throws SFSBStoreManagerException {       
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicatedSFSBStoreManager", "checkpointSave",
                new Object[] {beanState});
        }
        //added for monitoring
        EJBModuleStatistics stats = this.getEJBModuleStatistics();
        long saveStartTime = 0L;
        if(this.isMonitoringEnabled()) {
            saveStartTime = System.currentTimeMillis();
        }
        //end added for monitoring        
        StorePool storePool = this.getStorePool();
       
        ReplicatedEjbStore store = null;
        try {
            store = getStore();           
            ( (ReplicatedEjbStore) store).save(beanState, this);
        }
        catch (IOException e) {
            //e.printStackTrace();
            throw new SFSBStoreManagerException("Error during checkpointSave: id =" + beanState.getId().toString(), e);
        }
        finally {
            this.putStore(store);
        }
        //added for monitoring     
        if(this.isMonitoringEnabled()) {
            long saveEndTime = System.currentTimeMillis();
            stats.processCheckpointSave(saveEndTime - saveStartTime);
            stats.processBeanSize(this.getBeanSize(beanState));
        }
        //end added for monitoring        
       
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicatedSFSBStoreManager", "checkpointSave");
        }
    }
   
    /** Saves the state
     * @param state ReplicationState
     * @throws SFSBStoreManagerException
     */
    public void repairSave(ReplicationState state)
        throws SFSBStoreManagerException {
       
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicatedSFSBStoreManager", "repairSave",
                new Object[] {state});
        }
        //added for monitoring
        EJBModuleStatistics stats = this.getEJBModuleStatistics();
        long saveStartTime = 0L;
        if(this.isMonitoringEnabled()) {
            saveStartTime = System.currentTimeMillis();
        }
        //end added for monitoring        
        StorePool storePool = this.getStorePool();
       
        ReplicatedEjbStore store = null;
        try {
            store = getStore();           
            ( (ReplicatedEjbStore) store).saveForRepair(state, this);
        }
        catch (IOException e) {
            //e.printStackTrace();
            throw new SFSBStoreManagerException("Error during repairSave: id =" + state.getId().toString(), e);
        }
        finally {
            this.putStore(store);
        }
        //added for monitoring     
        if(this.isMonitoringEnabled()) {
            long saveEndTime = System.currentTimeMillis();
            stats.processCheckpointSave(saveEndTime - saveStartTime);
            //stats.processBeanSize(this.getBeanSize(beanState));
        }
        //end added for monitoring        
       
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicatedSFSBStoreManager", "repairSave");
        }
    }      

    /**
     * return the MonitorableSFSBStoreManager
     */   
    public MonitorableSFSBStoreManager getMonitorableSFSBStoreManager() {
        return this;
    }
   
    /** gets the state of the bean
     * @param id ID of the bean to be loaded from the HADB
     * @throws SFSBStoreManagerException
     * @return returns the bean with the state
     */
    public SFSBBeanState getState(Object id) throws SFSBStoreManagerException {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicatedSFSBStoreManager", "getState", id);
        }
        if(id == null) {
            return null;
        }
        SFSBBeanState sfsbState = null;
        //try first in locally passivated cache before going to backup
        sfsbState= this.getFromLocallyPassivatedCache(id.toString());
        if(sfsbState != null) {
            return sfsbState;
        }
       
        StorePool storePool = this.getStorePool();
        ReplicatedEjbStore store = null;
        try {
            store = getStore();
           
            sfsbState = store.loadBean(id);
            if(_logger.isLoggable(Level.FINE)) {
                _logger.exiting("ReplicatedSFSBStoreManager", "getStore", sfsbState);
            }
            return sfsbState;
           
        }
        catch (Exception e) {
            //e.printStackTrace();
            //throw e;           
            throw new SFSBStoreManagerException("Error loading SFSB state: id =" + id.toString(), e);
        }
        finally {
            this.putStore(store);
            if(_logger.isLoggable(Level.FINE)) {
                _logger.exiting("ReplicatedSFSBStoreManager", "getStore", sfsbState);
            }
            //return sfsbState;
        }
       
    }   
   
    /** Saves the state of the sfsb
     * @param beanState SFSBBeanState
     * @throws SFSBStoreManagerException
     */
    public void passivateSave(SFSBBeanState beanState)
        throws SFSBStoreManagerException {

        //store in local cache before pushing to backup
        putInLocallyPassivatedCache(beanState);
       
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicatedSFSBStoreManager", "passivateSave",
                new Object[] {beanState});
        }
        //added for monitoring
        EJBModuleStatistics stats = this.getEJBModuleStatistics();
        long saveStartTime = 0L;
        if(this.isMonitoringEnabled()) {
            saveStartTime = System.currentTimeMillis();
        }
        //end added for monitoring       
        StorePool storePool = this.getStorePool();
       
        ReplicatedEjbStore store = null;
        try {
            store = getStore();           
            ( (ReplicatedEjbStore) store).save(beanState, this);
        }
        catch (IOException e) {
            //e.printStackTrace();
            throw new SFSBStoreManagerException("Error during passivateSave: id =" + beanState.getId().toString(), e);
        }
        finally {
            this.putStore(store);
        }
        //added for monitoring     
        if(this.isMonitoringEnabled()) {
            long saveEndTime = System.currentTimeMillis();
            stats.processPassivateSave(saveEndTime - saveStartTime);
            stats.processBeanSize(this.getBeanSize(beanState));
        }
        //end added for monitoring        
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicatedSFSBStoreManager", "passivateSave");
        }
    }        

    /*
    public void remove(Object sessionKey) throws SFSBStoreManagerException {
    }
     */
   
    /** Removes the bean will be called when the client removes the bean
     * @param id ID of the bean to be removed
     */
    //FIXME reverting back to non-batch remove for testing
    public void remove(Object id) {
       
        this.removeIdFromLocallyPassivatedCache(id);       
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicatedSFSBStoreManager", "remove", id);
        }
        ReplicatedEjbStore store = null;
        try {
            store = getStore();
           
            store.remove(id);
            if(_logger.isLoggable(Level.FINE)) {
                _logger.exiting("ReplicatedSFSBStoreManager", "remove");
            }
           
        }
        catch (Exception e) {
            e.printStackTrace();
        } finally {
            this.putStore(store);
        }       
       
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicatedSFSBStoreManager", "remove");
        }
    }   
   
    /**
     * Remove all session data for this container
     * called during undeployment
     * @throws SFSBStoreManagerException
     */   
    public void removeAll() throws SFSBStoreManagerException {
      
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicatedSFSBStoreManager", "removeAll", containerId);
        }
       
        StorePool storePool = this.getStorePool();
        ReplicatedEjbStore store = null;
        try {
            store = getStore();
           
            store.undeployContainer();
            if(_logger.isLoggable(Level.FINE)) {
                _logger.exiting("ReplicatedSFSBStoreManager", "removeAll");
            }
        }
        catch (Exception e) {
            e.printStackTrace();
            throw new SFSBStoreManagerException("Error during ReplicatedSFSBStoreManager>>removeAll for container: " + containerId, e);
        }
        finally {
            //un-register with message router
            ReplicationMessageRouter router = null;
            if (Globals.IS_SECURITY_ENABLED) {
                router = (ReplicationMessageRouter)
                    AccessController.doPrivileged(
                        new PrivilegedGetReplicationMessageRouter());
            } else {
                router = ReplicationMessageRouter.createInstance();
            }
            if(router != null) {
                router.removeReplicationManager(this.getContainerID());
            }
            //stop background thread
            cleanupThread.threadStop();
            this.putStore(store);
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicatedSFSBStoreManager", "removeAll");
        }
        return;
       
    }
   
    /**
     * Remove all the idle/expired session data
     * that are idle for idleTimeoutInSeconds (passed during initSessionStore())
     * @throws SFSBStoreManagerException
     */
    public int removeExpiredSessions() throws SFSBStoreManagerException {
       
        //remove any expired beans from locally passivated cache too
        removeExpiredFromLocallyPassivated();
       
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicatedSFSBStoreManager", "removeExpired", containerId);
        }
        int result = 0;
        StorePool storePool = this.getStorePool();
        ReplicatedEjbStore store = null;
        try {
            store = getStore();
           
            result = store.removeExpiredSessions();
            if(_logger.isLoggable(Level.FINE)) {
                _logger.exiting("ReplicatedSFSBStoreManager", "removeExpired");
            }
        }
        catch (Exception e) {
            e.printStackTrace();
            throw new SFSBStoreManagerException("Error during ReplicatedSFSBStoreManager>>removeExpired for container: " + containerId, e);
        }
        finally {
            this.putStore(store);
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicatedSFSBStoreManager", "removeExpired");
        }
        return result;       
    }      
   
    /**
     * update only the lastAccessTime to the value time
     * Used when the session has been accessed as well
     * as periodically to keep session alive
     * @param sessionKey
     * @param time
     * @throws SFSBStoreManagerException
     */
    public void updateLastAccessTime(Object sessionKey, long time)
        throws SFSBStoreManagerException {

        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicatedSFSBStoreManager", "updateLastAccessTime", sessionKey.toString());
        }
       
        StorePool storePool = this.getStorePool();
        ReplicatedEjbStore store = null;
        try {
            store = getStore();
           
            store.updateLastAccessTime(sessionKey, time);
            if(_logger.isLoggable(Level.FINE)) {
                _logger.exiting("ReplicatedSFSBStoreManager", "updateLastAccessTime");
            }
        }
        catch (Exception e) {
            e.printStackTrace();
            throw new SFSBStoreManagerException("Error during ReplicatedSFSBStoreManager>>updateLastAccessTime for key: "
                + sessionKey + " errMsg: " + e.getMessage(), e);
        }
        finally {
            this.putStore(store);
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicatedSFSBStoreManager", "updateLastAccessTime");
        }
        return;             
    }   
   
    // SFSBTxStoreManager methods
   
    /**
     * @param beanStates
     * @throws SFSBStoreManagerException
     */
    public void checkpointSave(SFSBBeanState[] beanStates) throws SFSBStoreManagerException {
  
        //optimize if there is only a single beanState
        if(beanStates.length == 1) {
            this.checkpointSave(beanStates[0]);
            return;
        }
       
        EJBModuleStatistics stats = this.getEJBModuleStatistics();
        //FIXME: begin temp test code    
//        if (_logger.isLoggable(Level.FINE)) {
//            _logger.fine("TxCheckpointDurations Before checkpointSave:");
//            printTxCheckpointDurations(beanStates);
//        }
         // FIXME end test code
        long startTime = 0L;
        Long[] originalTxCheckpointDurations = {new Long(0L)};

        //added for monitoring
        if(this.isMonitoringEnabled()) {
            //temporarily store original txCheckpointDurations
            originalTxCheckpointDurations =
                this.storeOriginalTxCheckpointDurations(beanStates);
            startTime = System.currentTimeMillis();
        }
        //end added for monitoring
       
        /*
        _logger.entering("ReplicatedSFSBStoreManager", "checkpointSave",
        new Object[] {beanStates, new Boolean(isNew)});
         */
        StorePool storePool = this.getStorePool();
       
        ReplicatedEjbStore store = null;
        try {
            store = getStore();           
            ( (ReplicatedEjbStore) store).save(beanStates, startTime);
            //added for monitoring
            if(this.isMonitoringEnabled()) {
                for(int i=0; i<beanStates.length; i++) {
                    SFSBBeanState nextBeanState = beanStates[i];
                    stats.processBeanSize(this.getBeanSize(nextBeanState));
                }
            }
            //end added for monitoring          
        }
        catch (IOException e) {
            //e.printStackTrace();
            //added for monitoring
            if(this.isMonitoringEnabled()) {
                //restore original txCheckpointDurations
                this.restoreOriginalTxCheckpointDurations(beanStates, originalTxCheckpointDurations);
            }
            //end added for monitoring
            throw new SFSBStoreManagerException("Error during checkpointSave", e);
        }
        finally {
            this.putStore(store);
        }
        //FIXME: temp test code
        /*
        if (_logger.isLoggable(Level.FINE)) {
            _logger.fine("TxCheckpointDurations After checkpointSave:");
            printTxCheckpointDurations(beanStates);
        }
        */
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicatedSFSBStoreManager", "checkpointSave");
        }

    }
   
    private Long[] storeOriginalTxCheckpointDurations(SFSBBeanState[] beanStates) {   
        //return temporarily stored original txCheckpointDurations
        Long[] originalTxCheckpointDurations = new Long[beanStates.length];
        for(int i=0; i<beanStates.length; i++) {
            SFSBBeanState nextBeanState = beanStates[i];
            originalTxCheckpointDurations[i] =
                new Long(nextBeanState.getTxCheckpointDuration());
        }
        return originalTxCheckpointDurations;
    }
   
    private void restoreOriginalTxCheckpointDurations(SFSBBeanState[] beanStates, Long[] originalTxCheckpointDurations) {   
        //restore temporarily stored original txCheckpointDurations
        //to beanStates from originalTxCheckpointDurations
        for(int i=0; i<beanStates.length; i++) {
            SFSBBeanState nextBeanState = beanStates[i];
            long originalDuration = originalTxCheckpointDurations[i].longValue();
            nextBeanState.setTxCheckpointDuration(originalDuration);
        }
        return;
    }
   
    private void printTxCheckpointDurations(SFSBBeanState[] beanStates) {   
        //used for testing
        if (_logger.isLoggable(Level.FINE)) {
            _logger.fine("Printing Checkpoint Durations");
            for(int i=0; i<beanStates.length; i++) {
                SFSBBeanState nextBeanState = beanStates[i];
                _logger.fine("printTxCheckpointDurations for beanState[" + i + "]: "
                             + nextBeanState.getTxCheckpointDuration());
            }
        }
        return;
    }

    /**
     * gets the size (bytes) of the beanState
     * @param beanState
     * @return returns the size of the beanState
     */   
    protected long getBeanSize(SFSBBeanState beanState) {
        if (beanState == null) {
            return 0;
        }
        byte[] bytes = beanState.getState();
        if(bytes != null) {
            return bytes.length;
        } else {
            return 0;
        }
    }   
   
    // MonitorableSFSBStoreManager methods
   
    public void monitoringLevelChanged(boolean newValue) {
        //true means on -- false means off
        //FIXME do something
    }
   
    /**
    * return the current number of beans stored for this container
    */   
    public long getCurrentStoreSize() {
       
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicatedSFSBStoreManager", "getCurrentStoreSize");
        }
        int result = 0;
        //StorePool storePool = this.getStorePool();
        ReplicatedEjbStore store = null;
        try {
            store = getStore();
           
            result = store.getContainerSize();
            if(_logger.isLoggable(Level.FINE)) {
                _logger.exiting("ReplicatedSFSBStoreManager", "getCurrentStoreSize");
            }
        }
        catch (Exception e) {
            e.printStackTrace();
        }
        finally {
            this.putStore(store);
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicatedSFSBStoreManager", "getCurrentStoreSize");
        }
        return result;           
       
    }
   
    public void repair(long repairStartTime) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicatedSFSBStoreManager>>repair");
        }        
        /*
        if (!started)
            return;
         */
        if(ReplicationHealthChecker.isStopping()) {
            return;
        }       

        SFSBBeanState beanStates[] = getLocallyPassivatedBeanStatesArray();
        for (int i = 0; i < beanStates.length; i++) {
            SFSBBeanState beanState = (SFSBBeanState) beanStates[i];
            if(beanState.getLastAccess() <= repairStartTime) {
                try {
                    checkpointSave(beanState);
                } catch(SFSBStoreManagerException ex) {
                     // FIXME evaluate log level
                    if (_logger.isLoggable(Level.FINE)) {
                        _logger.log(Level.FINE, "Exception occurred during force flush", ex);
                    }
                } catch(Throwable t) {
                    // FIXME evaluate log level
                    if (_logger.isLoggable(Level.FINE)) {
                        _logger.log(Level.FINE, "Throwable exception occurred during force flush", t);
                    }
                    break;
                }
            }
        }
    }
   
    public void repair(long repairStartTime, boolean checkForStopping) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicatedSFSBStoreManager>>repair");
        }
        /*
        if (!started)
            return;
         */
        if(checkForStopping && ReplicationHealthChecker.isStopping()) {
            return;
        }       

        SFSBBeanState beanStates[] = getLocallyPassivatedBeanStatesArray();
        for (int i = 0; i < beanStates.length; i++) {
            SFSBBeanState beanState = (SFSBBeanState) beanStates[i];
            if(beanState.getLastAccess() <= repairStartTime) {
                try {
                    //_logger.info("repair checkpointing beanid: " + beanState.getId());
                    checkpointSave(beanState);
                } catch(SFSBStoreManagerException ex) {
                    // FIXME evaluate log level
                    if (_logger.isLoggable(Level.FINE)) {
                        _logger.log(Level.FINE, "Exception occurred during force flush", ex);
                    }
                } catch(Throwable t) {
                    // FIXME evaluate log level
                    if (_logger.isLoggable(Level.FINE)) {
                        _logger.log(Level.FINE, "Exception occurred during force flush", t);
                    }
                    break;
                }
            }
        }
    }
   
    public void respondToFailure(String instanceName, boolean checkForStopping) {
        ; //no-op
    }
   
    public SFSBBeanState[] getLocallyPassivatedBeanStatesArray() {
       
        BaseCache passivatedSessions = locallyPassivatedSessions;
        SFSBBeanState[] beanStates = null;
        int numberOfIds = passivatedSessions.getEntryCount();
        ArrayList valuesList = new ArrayList(numberOfIds);
        Iterator valuesIter = passivatedSessions.values();
        while(valuesIter.hasNext()) {
            valuesList.add((SFSBBeanState)valuesIter.next());
        }
        SFSBBeanState[] template = new SFSBBeanState[valuesList.size()];
        beanStates = (SFSBBeanState[])valuesList.toArray(template);
        return beanStates;

    }
   
    /**
     * remove any expired states from the locally passivated cache
     */
    public void removeExpiredFromLocallyPassivated() {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicatedSFSBStoreManager>>removeExpiredFromLocallyPassivated");
        }       
        ArrayList expiredLocallyPassivated = new ArrayList(30);
        BaseCache locallyPassivatedCache = this.getLocallyPassivatedSessions();
        for(Iterator it = locallyPassivatedCache.values(); it.hasNext();) {
            SFSBBeanState nextBeanState = (SFSBBeanState)it.next();
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("ReplicatedSFSBStoreManager>>removeExpiredFromLocallyPassivated:nextState=" + nextBeanState);
            }
            if(isBeanExpired(nextBeanState)) {
                expiredLocallyPassivated.add(nextBeanState);
            }
        }
        if(_logger.isLoggable(Level.FINE)) {          
            _logger.fine("removeExpiredFromLocallyPassivated:expiredReplicas.size=" + expiredLocallyPassivated.size());
        }               
        for(int i=0; i<expiredLocallyPassivated.size(); i++) {
            SFSBBeanState nextBeanState =
                (SFSBBeanState)expiredLocallyPassivated.get(i);
            this.removeFromLocallyPassivatedCache(nextBeanState);
        }       
    }
   
    private boolean isBeanExpired(SFSBBeanState beanState) {
        long timeAlive = System.currentTimeMillis() - beanState.getLastAccess();
        return (timeAlive > (this.getIdleTimeoutInSeconds()* 1000L));       
    }
   
    //receive processing methods
   
    /**
     * process the save of the replicationState
     * @param message
     */    
    public void processValvesave(ReplicationState message) {
        if(_logger.isLoggable(Level.FINER)) {
            _logger.entering("ReplicatedSFSBStoreManager", "processValvesave", message);
                   
        }
        if (_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicatedSFSBStoreManager>>processValvesave :appId=" + message.getAppId() +
                         " :sessionId=" + message.getId() + " :version="  + message.getVersion());
        }
        this.putInReplicationCache(message);
    }

    /**
     * process the remove of the replicationState
     * @param message
     */    
    public void processRemove(ReplicationState message) {
        if(message.getState() != null) {
            processBulkRemove(message);
            return;
        }
        replicaCache.processRemove(message);
    }

    public void processBulkRemove(ReplicationState message) {
        replicaCache.removeFromReplicationCache(message.getState());
    }
    /**
     * process the removeExpired for this container
     * @param message
     */    
    public void processRemoveExpired(ReplicationState message) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicatedSFSBStoreManager>>processRemoveExpired");
        }       
        //FIXME finish this
        ArrayList expiredReplicas = new ArrayList(30);
        BaseCache replicasCache = this.getReplicatedSessions();
        for(Iterator it = replicasCache.values(); it.hasNext();) {
            ReplicationState nextState = (ReplicationState)it.next();
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("ReplicatedSFSBStoreManager>>processRemoveExpired:nextState=" + nextState);
            }           
            if(nextState.isExpired()) {
                expiredReplicas.add(nextState);
            }
        }
        if(_logger.isLoggable(Level.FINE)) {          
            _logger.fine("processRemoveExpired:expiredReplicas.size=" + expiredReplicas.size());
        }               
        for(int i=0; i<expiredReplicas.size(); i++) {
            ReplicationState nextState =
                (ReplicationState)expiredReplicas.get(i);
            this.removeFromReplicationCache(nextState);
        }       
    }

    /**
     * process the updateLastAccessTime of the replicationState
     * @param message
     */   
    public void processUpdatelastaccesstime(ReplicationState message) {
        replicaCache.processUpdatelastaccesstime(message);
    }   
   
    //query processing methods
   
    public ReplicationState processBroadcastfindsessionPrevious(ReplicationState queryState) {
        //FIXME complete query and send back response
        //_logger.info("ReplicatedSFSBStoreManager>>processBroadcastfindsession:instance: " + getInstanceName());       
        ReplicationState replicaState
            = findReplicatedState(queryState);
        ReplicationState returnState = null;
        if(replicaState != null) {
            returnState = ReplicationState.createQueryResponseFrom(replicaState);
        }
        //_logger.info("ReplicatedSFSBStoreManager>>processBroadcastfindsession:returnState=" + returnState);
        /*
        //temp code returning from local cache - decide if that's ok
        if(returnState == null) {
            Session sess = null;
            try {
                sess = this.findSessionFromCacheOnly((String)queryState.getId());
            } catch (IOException ex) {}
            if(sess != null) {
                returnState = createQueryResponse(sess);          
            }
            //_logger.info("ReplicatedSFSBStoreManager>>processBroadcastfindsession:localCachedBean=" + sess);           
        }
        //end temp code
         */
        return returnState;

    }
   
    public ReplicationState processBroadcastfindsession(ReplicationState queryState) {
        //complete query and send back response
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindSession:instance: " + getInstanceName());
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindSession:id=" + queryState.getId());
        }       
        ReplicationState returnState = null;
        //if queryVersion == -1, then this is version unaware load
        long queryVersion = queryState.getVersion();
        ReplicationState replicaState
            = findReplicatedState(queryState);
        //first check for none found
        if(replicaState == null) {
            //return nack
            return ReplicationState.createQueryResponseFrom(queryState, true);
        }
        //next check for stale version if version-aware query
        if(queryVersion != -1 && replicaState.getVersion() < queryVersion) {
            this.removeFromReplicationCache(replicaState);
            //return nack
            return ReplicationState.createQueryResponseFrom(replicaState, true);           
        }
        //at this point we know replicaState is not null
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicatedSFSBStoreManager>>processBroadcastfindsession:REPLICA_FOUND:replicaStateVersion:" + replicaState.getVersion());
            _logger.fine("ReplicatedSFSBStoreManager>>processBroadcastfindsession:REPLICA_FOUND:replicaState:" + replicaState.getTrunkState());
            _logger.fine("ReplicatedSFSBStoreManager>>processBroadcastfindsession:REPLICA_FOUND:replicaAttrState" + replicaState.getState());             
        }
        returnState = ReplicationState.createQueryResponseFrom(replicaState);
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicatedSFSBStoreManager>>processBroadcastfindsession:replicaStateResponseVersion:" + returnState.getVersion());             
        }
        //FIXME may want to wait for ack before doing this      
        //replicatedSessions.remove(replicaState.getId());
        //while here check and remove from manager cache if present
        this.clearFromPassivatedCache((String)queryState.getId());

        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindSession:returnState=" + returnState);             
        }        
        return returnState;

    }
   
    public void processBroadcastloadreceived(ReplicationState queryState) {
        //load is acknowledged safe to remove replica now
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastloadreceived:instance: " + getInstanceName());
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastloadreceived:id=" + queryState.getId());                       
        }
        if(queryState == null || queryState.getId() == null) {
            return;
        }

        String ignoreInstance = (String) queryState.getProperty(ReplicationState.IGNORE_REMOVE_INSTANCE_NAME);
        //only safe to remove replica if we are not the replica partner
        //now determined by passed property
        if(ignoreInstance != null && !ignoreInstance.equals(ReplicationUtil.getInstanceName())) {
            removeFromReplicationCache((String)queryState.getId());
        }
    }        
   
    protected ReplicationState findReplicatedState(ReplicationState queryState) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("findReplicatedState:id = " + queryState.getId());             
        }             
        return this.getFromReplicationCache( (String)queryState.getId() );
    }
   
    protected void clearFromPassivatedCache(String id) {
        SFSBBeanState beanState = this.getFromLocallyPassivatedCache(id);
        if(beanState != null) {                              
            this.removeFromLocallyPassivatedCache(beanState);
        }
    }   
   
    // message processing

    public void processMessage(ReplicationState message) {
        //FIXME complete
        if(ReplicationState.isBroadcastState(message)) {
            processBroadcastMessage(message);
            return;
        }
       
        ReplicationState response =
            ReplicationState.createResponseFrom(message);
        //send a response before further processing only if processed
        //msg is not itself a response and if method is a void return type
        //FIXME this send will be removed if upstream ack send works
        /* removing this for test
        if(!message.isReturnMessage() && message.isVoidMethodReturnState()) {
            try {
                this.doSendResponse(response);
            } catch (SFSBStoreManagerException ex) {
                //FIXME log this
            }            
        }
         */       

        boolean isResponse = this.doProcessMessage(message);
        //send a response only if processed msg is not itself a response
        //and if method is not void return type (in that case ack was
        //already sent)
        if(!isResponse && !message.isVoidMethodReturnState()) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("ReplicatedSFSBStoreManager>>processMessage -sending response:command: " + message.getCommand());             
            }
            /*
            ReplicationState response =
                ReplicationState.createResponseFrom(message);
             */
            try {
                this.doSendResponse(response);
            } catch (SFSBStoreManagerException ex) {
                //FIXME log this
            }           
        }
    }
   
    //return true if message is processResponse
    public boolean doProcessMessage(ReplicationState message) {
        //FIXME complete
        boolean result = false;
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in ReplicatedSFSBStoreManager>>doProcessMessage");             
        }       
        String methodName = getProcessMethodName(message);
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in ReplicatedSFSBStoreManager>>doProcessMessage:methodName=" + methodName);             
        }       
        try {
            Class myClass = this.getClass();
            myClass.getMethod(
                methodName,
                    new Class[]{ message.getClass() }).invoke(
                        this, new Object[]{ message });           
        } catch (IllegalAccessException ex) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("in ReplicatedSFSBStoreManager>>doProcessMessage:methodName=" + methodName + "illegalAccessException");             
            }           
        } catch (NoSuchMethodException ex) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("in ReplicatedSFSBStoreManager>>doProcessMessage:methodName=" + methodName + "noSuchMethodException");             
            }            
        } catch (InvocationTargetException ex) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("in ReplicatedSFSBStoreManager>>doProcessMessage:methodName=" + methodName + "invocationTargetException");
                _logger.fine("invocationException.getCause()= " + ex.getCause());
            }                     
            ex.printStackTrace();
        }
        if(methodName.equals("processResponse")) {
            result = true;
        }
        return result;
    }
   
    private String getProcessMethodName(ReplicationState message) {
        String command = message.getCommand();
        return "process" + camelCase(command);
    }
   
    /**
     * this method strips out all non-alpha characters; camelCases the result
     *
     * @param inputString
     */    
    private String camelCase(String inputString) {
        String strippedString = stripNonAlphas(inputString);
        String firstLetter = (strippedString.substring(0, 1)).toUpperCase();
        String remainingPart =
            (strippedString.substring(1, strippedString.length())).toLowerCase();
        return firstLetter + remainingPart;
    }
   
    /**
     * this method strips out all non-alpha characters
     *
     * @param inputString
     */    
    private String stripNonAlphas(String inputString) {
        StringBuffer sb = new StringBuffer(50);
        for(int i=0; i<inputString.length(); i++) {
            char nextChar = inputString.charAt(i);
            if(Character.isLetter(nextChar)) {
                sb.append(nextChar);
            }
        }
        return sb.toString();
    }
   
    public void processResponse(ReplicationState message) {
        //complete processing response - not sending response to a response
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in ReplicationWebEventPersistentManager>>processResponse");
        }              
        ReplicationResponseRepository.putEntry(message);
    }
   
    public ReplicationState processBroadcastresponse(ReplicationState queryResponseState) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastresponse:queryResponseState=" + queryResponseState);           
        }
        //ReplicationResponseRepository.putEntry(queryResponseState);
        ReplicationResponseRepository.putFederatedEntry(queryResponseState);
        return queryResponseState;
    }   
   
    public void processBroadcastMessage(ReplicationState message) {
        //FIXME complete
        ReplicationStateQueryResponse response = this.doProcessQueryMessage(message);
        boolean isResponse = response.isResponse();
        ReplicationState responseState = response.getState();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("processBroadcastMessage:after doProcessQueryMessage:response=" + isResponse);
            _logger.fine("processBroadcastMessage:after doProcessQueryMessage:responseState=" + responseState);           
        }       
        //don't send a response to a response
        if(!isResponse) {
            //FIXME point-to-point response back to sender
           try {
                doSendQueryResponse(responseState, this.getInstanceName());
            } catch (SFSBStoreManagerException ex) {
                //FIXME evaluate log level
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.log(Level.FINE, "exception occurred in ReplicatedSFSBStoreManager>>processBroadcastMessage",
                                  ex);
                }
            }
            /*
            ReplicationState response =
                ReplicationState.createResponseFrom(message);
            this.doSendResponse(response);
             */
        }
    }
   
    public void processQueryMessage(ReplicationState message, String returnInstance) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicatedSFSBStoreManager>>processQueryMessage:returnInstance= " + returnInstance);
        }       
        ReplicationStateQueryResponse response = this.doProcessQueryMessage(message);
        boolean isResponse = response.isResponse();
        ReplicationState responseState = response.getState();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("processQueryMessage:after doProcessQueryMessage:response=" + isResponse);
            _logger.fine("processQueryMessage:after doProcessQueryMessage:responseState=" + responseState);           
        }        
        //don't send a response to a response
        if(!isResponse && responseState != null) {
            //FIXME point-to-point response back to sender
            //_logger.fine("processQueryMessage - need to send back result to " + returnInstance);
            try {
                //doSendQueryResponse(responseState, this.getInstanceName());
                doSendQueryResponse(responseState, returnInstance);
            } catch (SFSBStoreManagerException ex) {
                //FIXME log this
            }           
        }
    }    
   
    //return true if message is processQueryResponse
    public ReplicationStateQueryResponse doProcessQueryMessage(ReplicationState message) {
        ReplicationState resultState = null;
        String methodName = getProcessMethodName(message);
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in ReplicatedSFSBStoreManager>>doProcessQueryMessage:methodName=" + methodName);
        }       
        try {
            Class myClass = this.getClass();
            resultState = (ReplicationState) myClass.getMethod(
                methodName,
                    new Class[]{ message.getClass() }).invoke(
                        this, new Object[]{ message });           
        } catch (IllegalAccessException ex) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("in ReplicatedSFSBStoreManager>>doProcessQueryMessage:methodName=" + methodName + "illegalAccessException");             
            }                       
        } catch (NoSuchMethodException ex) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("in ReplicatedSFSBStoreManager>>doProcessQueryMessage:methodName=" + methodName + "noSuchMethodException");             
            }                       
        } catch (InvocationTargetException ex) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("in ReplicatedSFSBStoreManager>>doProcessQueryMessage:methodName=" + methodName + "invocationTargetException");
                _logger.fine("invocationException.getCause()= " + ex.getCause());
            }            
            ex.printStackTrace();
        }
        boolean isResponse = methodName.equals("processBroadcastresponse");
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicationWebEventPresistentManager>>doProcessQueryMessage:resultState=" + resultState);
        }
        if (resultState != null) {
            resultState.setRouteAdvertisement(message.getRouteAdvertisement());
        }       
        return new ReplicationStateQueryResponse(resultState, isResponse);
    }     
   
    /**
    * send the response
    *
    * @param sessionState
    *   The replication state response
    */   
    public void doSendResponse(ReplicationState sessionState)
        throws SFSBStoreManagerException {
       
        StorePool storePool = this.getStorePool();
        ReplicatedEjbStore store = null;
        try {
            store = getStore();
            store.sendResponse(sessionState);
            if(_logger.isLoggable(Level.FINE)) {
                _logger.exiting("ReplicatedSFSBStoreManager", "doSendResponse");
            }           
        }
        catch (Exception e) {
            //e.printStackTrace();
            //throw e;           
            throw new SFSBStoreManagerException("Error sending ReplicationState response: id =" + sessionState.getId().toString(), e);
        }
        finally {
            this.putStore(store);
            if(_logger.isLoggable(Level.FINE)) {
                _logger.exiting("ReplicatedSFSBStoreManager", "doSendResponse");
            }
        }       
    }
   
    /**
    * send the response
    *
    * @param sessionState the replication state response
    * @param instanceName  the name of the target instance
    */   
    public void doSendQueryResponse(ReplicationState sessionState, String instanceName)
        throws SFSBStoreManagerException {
       
        StorePool storePool = this.getStorePool();
        ReplicatedEjbStore store = null;
        try {
            store = getStore();
            store.sendQueryResponse(sessionState, instanceName);
            if(_logger.isLoggable(Level.FINE)) {
                _logger.exiting("ReplicatedSFSBStoreManager", "doSendQueryResponse");
            }           
        }
        catch (Exception e) {
            //e.printStackTrace();
            //throw e;           
            throw new SFSBStoreManagerException("Error sending ReplicationState query response: id =" + sessionState.getId().toString(), e);
        }
        finally {
            this.putStore(store);
            if(_logger.isLoggable(Level.FINE)) {
                _logger.exiting("ReplicatedSFSBStoreManager", "doSendQueryResponse");
            }
        }       
    }
   
    // begin post join reconciliation

    /**
     * do reconciliation processing
     * used for both rolling upgrade and post network partition rejoin
     * @param waitTime the waitTime in seconds
     * @param ctx the RollingUpgradeContext
     */
    public void doPostJoinReconciliation(long waitTime, RollingUpgradeContext ctx) {
        long startTime = 0L;
        if (_logger.isLoggable(Level.FINE)) {
            startTime = System.currentTimeMillis();
        }
        CountDownLatch doneSignal = new CountDownLatch(2);
        //do active cache reconciliation
        ReconcileActive reconcileActive
            = new ReconcileActive((ReplicationManager)this, waitTime, doneSignal, ctx);
        RollingUpgradeHandler.executeTask(reconcileActive);

        //trigger replica cache reconciliation
        TriggerReconcileReplica triggerReconcileReplica
            = new TriggerReconcileReplica((ReplicationManager)this, waitTime, doneSignal, ctx);
        RollingUpgradeHandler.executeTask(triggerReconcileReplica);

        try {
            doneSignal.await(waitTime, TimeUnit.SECONDS);
        } catch(InterruptedException ex) {
            ;
        } finally {
            if(doneSignal.getCount() != 0) {
                String errorMsg = "ReplicatedSFSBStoreManager>>doPostJoinReconciliation timed out after " + waitTime + " seconds";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicatedSFSBStoreManager doPostJoinReconciliation successful after a wait: wait time = " + (System.currentTimeMillis() - startTime));
                }
            }
        }
    }

    // end post join reconciliation  
   
   // begin rolling upgrade related code
   
    public void doRollingUpgradePreShutdownProcessing(long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        //do syncpoint save
        doSyncpointSave(waitTime, doneSignal, ctx);
    }
   
    public void doRollingUpgradePostStartupProcessing(String rollingUpgradeType, long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        //do syncpoint load
        // TODO :: implement memory based restore.
        doSyncpointLoad(waitTime, doneSignal, ctx);
    }

    public void doRollingUpgradePostStartupReconciliationProcessing(long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        //do post join reconciliation
        try {
            doPostJoinReconciliation(waitTime, ctx);
        } finally {
            doneSignal.countDown();
        }
    }

    void doSyncpointSave(long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        try {
            if(this.isSkipRollingUpgradeBackupRestore()) {
                return;
            }
            EJBFileSync syncStore = new EJBFileSync((ReplicationManager)this);
            try {
                syncStore.save(waitTime, ctx);
            } catch (IOException ex) {
                ex.printStackTrace();
                String errorMsg = "ReplicatSFSBStoreManager>>doSyncPointSave IOException";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            }
        } finally {
            doneSignal.countDown();
        }
    }

    void doSyncpointLoad(long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        try {
            if(this.isSkipRollingUpgradeBackupRestore()) {
                return;
            }
            WebFileSync syncStore = new WebFileSync((ReplicationManager)this);
            try {
                syncStore.load(waitTime, ctx);
            } catch (IOException ex) {
                String errorMsg = "ReplicatSFSBStoreManager>>doSyncPointLoad IOException";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } catch (ClassNotFoundException ex2) {
                String errorMsg = "ReplicatSFSBStoreManager>>doSyncPointLoad ClassNotFoundException";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            }
        } finally {
            doneSignal.countDown();
        }
    }

    void doMemoryLoad(long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        try {
            if(this.isSkipRollingUpgradeBackupRestore()) {
                return;
            }
            EJBMemorySync syncStore = new EJBMemorySync((ReplicationManager)this);
            try {
                syncStore.load(waitTime, ctx);
                //todo fix next line
                //markActiveCacheAsSuspected(true);
            } catch (IOException ex) {
                String errorMsg = "ReplicatSFSBStoreManager>>doMemoryLoad IOException";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } catch (ClassNotFoundException ex2) {
                String errorMsg = "ReplicatSFSBStoreManager>>doMemoryLoad ClassNotFoundException";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            }
        } finally {
            doneSignal.countDown();
        }
    }

    void restoreActiveCacheViaMemory(long waitTime, RollingUpgradeContext ctx) {
        //todo
        triggerActiveCacheRestoration(waitTime, ctx);
    }

    void restoreReplicaCacheViaMemory(long waitTime, RollingUpgradeContext ctx) {
        //todo
    }

    public void triggerActiveCacheRestoration(long waitTime, RollingUpgradeContext ctx) {
        long startTime = 0L;
        if (_logger.isLoggable(Level.FINE)) {
            startTime = System.currentTimeMillis();
        }
        int numberOfRespondants = RollingUpgradeUtil.getNumberOfExpectedRespondants();
        restoreActiveCacheDoneSignal = new CountDownLatch(numberOfRespondants);
        //trigger replicateTo partner(s) to do
        //active cache restoration
        ownedReplicaListsReceivedForActiveCacheRestoration
            = new ConcurrentHashMap<String, ReplicationState>();
        doTriggerActiveCacheRestoration(ReplicationUtil.checkIsInstanceLoadBalancedByCLB());

        try {
            restoreActiveCacheDoneSignal.await(waitTime, TimeUnit.SECONDS);
            processActiveCacheRestorationResults(ownedReplicaListsReceivedForActiveCacheRestoration);
        } catch(InterruptedException ex) {
            ;
        } finally {
            if(restoreActiveCacheDoneSignal.getCount() != 0) {
                String errorMsg = "ReplicationManagerBase>>triggerActiveCacheRestoration timed out after " + waitTime + " seconds";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationManagerBase triggerActiveCacheRestoration successful after a wait: wait time = " + (System.currentTimeMillis() - startTime));
                }
            }
        }
    }

    protected void doTriggerActiveCacheRestoration(boolean isInstanceLoadBalancedByCLB) {
        if(!isInstanceLoadBalancedByCLB) {
            //trigger replicateTo partner to do
            //active cache restoration
            ReplicationHealthChecker healthChecker
                = ReplicationHealthChecker.getInstance();
            String replicateToInstanceName = healthChecker.getReshapeReplicateToInstanceName(null, 0L);
            sendRollingUpgradeActiveCacheRestorationAdvisory(replicateToInstanceName);
        } else {
            //trigger all partners to do
            //active cache restoration
            sendRollingUpgradeActiveCacheRestorationAdvisory();
        }
    }

    protected void processActiveCacheRestorationResults(ConcurrentHashMap<String, ReplicationState> ownedReplicaListsReceivedForActiveCacheRestoration) {
        ReplicatedEjbStore store = getSingletonStore();
        Collection<ReplicationState> ownedReplicasList = ownedReplicaListsReceivedForActiveCacheRestoration.values();
        Iterator<ReplicationState> it = ownedReplicasList.iterator();
        while(it.hasNext()) {
            ReplicationState nextOwnedListReplicationState = it.next();
            List<ReplicationState>nextOwnedListOfStates
                = (List<ReplicationState>)RollingUpgradeUtil.getObject(nextOwnedListReplicationState.getState());
            for(int i=0; i<nextOwnedListOfStates.size(); i++) {
                ReplicationState nextState = nextOwnedListOfStates.get(i);
                SFSBBeanState nextBeanState = null;
                try {
                    nextBeanState = store.createBeanState(nextState);
                } catch (Exception ex) {}
                if(nextBeanState != null) {
                    //todo need to add beanState equivalents into the container active cache
                    //this.add(nextBeanState);
                }
            }
        }
    }
   
    public void doActiveCacheReconciliation(long waitTime, RollingUpgradeContext ctx) {
        //System.out.println("begin doActiveCacheReconciliation");
        if(!this.isThirdPartyBackingStoreInUse()) {
            stillOwnedEjbIdsForActiveCacheReconciliation
                = triggerGetIdsForActiveCacheReconciliation(waitTime);
        } else {
            List sessionIdsList = getEjbIdsThirdPartySPI(ReplicationUtil.getInstanceName());
            stillOwnedEjbIdsForActiveCacheReconciliation
                = RollingUpgradeUtil.mergeQueryListIntoMap(sessionIdsList, stillOwnedEjbIdsForActiveCacheReconciliation);
        }
        this.setActiveCacheReconciliationOngoing(true);
        try {
            reconcileSessionActiveCache(stillOwnedEjbIdsForActiveCacheReconciliation);
        } finally {
            this.setActiveCacheReconciliationOngoing(false);
            //System.out.println("end doActiveCacheReconciliation");
        }
    }

    public ConcurrentHashMap<String, FederatedQueryListElement> triggerGetIdsForActiveCacheReconciliation(long waitTime) {
        long startTime = 0L;
        if (_logger.isLoggable(Level.FINE)) {
            startTime = System.currentTimeMillis();
        }
        //because sending to all instances
        int numberOfRespondants = ReplicationUtil.getNumberExpectedRespondants();
        getIdsForActiveCacheReconciliationDoneSignal = new CountDownLatch(numberOfRespondants);
        //trigger replicateTo partner(s) to
        //provide owned session ids list(s)
        ownedIdsForActiveCacheReconciliation
            = new ConcurrentHashMap<String, ReplicationState>();       
        doTriggerGetIdsForActiveCacheReconciliation(ReplicationUtil.checkIsInstanceLoadBalancedByCLB());
        ConcurrentHashMap<String, FederatedQueryListElement> result = null;
        try {
            getIdsForActiveCacheReconciliationDoneSignal.await(waitTime, TimeUnit.SECONDS);
            result = processCacheReconciliationResults(ownedIdsForActiveCacheReconciliation);
        } catch(InterruptedException ex) {
            result = new ConcurrentHashMap<String, FederatedQueryListElement>();
        } finally {
            if(getIdsForActiveCacheReconciliationDoneSignal.getCount() != 0) {
                _logger.log(Level.WARNING, "ReplicatedSFSBStoreManager>>triggerGetIdsForActiveCacheReconciliation timed out after "
                    + waitTime + " seconds");
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicatedSFSBStoreManager triggerGetIdsForActiveCacheReconciliation successful after a wait: wait time = " + (System.currentTimeMillis() - startTime));
                }
            }
            ownedIdsForActiveCacheReconciliation = null;
        }
        return result;
    }

    protected void doTriggerGetIdsForActiveCacheReconciliation(boolean isInstanceLoadBalancedByCLB) {
        //trigger all partners to do
        //active cache reconciliation
        sendRollingUpgradeGetIdsForActiveCacheReconciliationAdvisory();
    }

    /**
     * broadcast a rolling upgrade getIds
     * for replica cache reconciliation advisory message
     *
     */
    protected void sendRollingUpgradeGetIdsForActiveCacheReconciliationAdvisory() {
        //broadcast rolling upgrade getIds for replica cache reconciliation advisory
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_EJB, MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_ADVISORY);
    }

    public void processRollingupgradegetidsforactivecachereconciliationadvisory(ReplicationState replicationState) {
        String instanceName = ReplicationUtil.getInstanceName();
        String owningInstanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processRollingupgradegetidsforactivecachereconciliationadvisory:instance: " + instanceName);
            _logger.fine("in " + this.getClass().getName() + ">>processRollingupgradegetidsforactivecachereconciliationadvisory:owningInstance=" + owningInstanceName);
        }
        List<FederatedQueryListElement> sessionIds = getEjbIds(owningInstanceName);
        sendReplicasList(sessionIds, owningInstanceName, MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_RESPONSE);
    }

    public void processRollingupgradegetidsforactivecachereconciliationresponse(ReplicationState replicationState) {
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradegetidsforactivecachereconciliationresponse received from " + instanceName);
        }
        ownedIdsForActiveCacheReconciliation.putIfAbsent(instanceName, replicationState);
        getIdsForActiveCacheReconciliationDoneSignal.countDown();
    }
   
    /**
     * reconcile the active cache with a map containing
     * current set of instance ids owned by this instance
     * @param stillOwnedIds
     */   
    private void reconcileSessionActiveCache(ConcurrentHashMap stillOwnedIds) {
        // FIXME
    }

    protected void doTriggerReplicaCacheReconciliation(long waitTime, boolean isInstanceLoadBalancedByCLB) {
        if(!isInstanceLoadBalancedByCLB) {
            //trigger replicateFrom partner to do
            //replica cache reconciliation
            ReplicationHealthChecker healthChecker
                = ReplicationHealthChecker.getInstance();
            String replicateFromInstanceName = healthChecker.getReshapeReplicateFromInstanceName();
            sendRollingUpgradeAdvisory(waitTime, replicateFromInstanceName);
        } else {
            //trigger all partners to do
            //replica cache reconciliation
            sendRollingUpgradeAdvisory(waitTime);
        }
    }  

    public void triggerReplicaCacheReconciliation(long waitTime, RollingUpgradeContext ctx) {
        int numberOfRespondants = 1;
        if(ReplicationUtil.checkIsInstanceLoadBalancedByCLB()) {
            numberOfRespondants = ReplicationUtil.getNumberExpectedRespondants();
        }
        long startTime = System.currentTimeMillis();
        if (_logger.isLoggable(Level.FINE)) {
            startTime = System.currentTimeMillis();
        }
        reconcileReplicaCacheDoneSignal = new CountDownLatch(numberOfRespondants);
        //trigger replicateFrom partner(s) to do
        //replica cache reconciliation
        doTriggerReplicaCacheReconciliation(waitTime, ReplicationUtil.checkIsInstanceLoadBalancedByCLB());
        try {
            reconcileReplicaCacheDoneSignal.await(waitTime, TimeUnit.SECONDS);
        } catch(InterruptedException ex) {
            ;
        } finally {
            if(reconcileReplicaCacheDoneSignal.getCount() != 0) {
                String errorMsg = "ReplicatedSFSBStoreManager>>triggerReplicaCacheReconciliation timed out after " + waitTime + " seconds";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicatedSFSBStoreManager triggerReplicaCacheReconciliation successful after a wait: wait time = " + (System.currentTimeMillis() - startTime));
                }
            }
        }
    }

    protected ConcurrentHashMap<String, FederatedQueryListElement> processCacheReconciliationResults(ConcurrentHashMap<String, ReplicationState> ownedIdsForReplicaCacheReconciliation) {
        ConcurrentHashMap<String, FederatedQueryListElement> result =
            new ConcurrentHashMap<String, FederatedQueryListElement>();
        Collection<ReplicationState> ownedReplicasList = ownedIdsForReplicaCacheReconciliation.values();
        Iterator<ReplicationState> it = ownedReplicasList.iterator();
        while(it.hasNext()) {
            ReplicationState nextOwnedListReplicationState = it.next();
            List<FederatedQueryListElement>nextOwnedListOfStates
                = (List<FederatedQueryListElement>)RollingUpgradeUtil.getObject(nextOwnedListReplicationState.getState());
            for(int i=0; i<nextOwnedListOfStates.size(); i++) {
                FederatedQueryListElement nextElement = nextOwnedListOfStates.get(i);
                String nextElementId = nextElement.getId();
                if(nextElementId != null) {
                    FederatedQueryListElement existingElement = result.get(nextElementId);
                    if(existingElement == null || existingElement.getVersion() < nextElement.getVersion()) {
                        result.put(nextElementId, nextElement);
                    }
                }
            }
        }
        return result;
    }   

    /**
     * reconcile the replica cache (of your replica partner)
     * query instance1 to get a list of replica id/version data elements
     * then do 2 iterations:
     * iterate over the query result:
     * if an id from this list does not exist in this active cache
     * issue a remove message & load acknowledgment
     * if an id exists and the versions match do nothing
     * if an id exists and the active version is > replica version,
     *   - do a save
     * iterate over the active cache
     * if an id from active cache does not exist in the replica list
     *   - do a save
     * @param waitTime the waitTime
     */
    public void doReplicaCacheReconciliation(long waitTime, RollingUpgradeContext ctx) {
        //System.out.println("begin doReplicaCacheReconciliation");
        //todo - check that zombies are cleaned out during execution of query that follows
        //if so cleanOutZombieReplicas can be removed
        cleanOutZombieReplicas();
        ConcurrentHashMap<String, FederatedQueryListElement> stillOwnedEjbIdsForReplicaCacheReconciliation
            = null;
        if(!this.isThirdPartyBackingStoreInUse()) {
            stillOwnedEjbIdsForReplicaCacheReconciliation
                = triggerGetIdsForReplicaCacheReconciliation(waitTime);
        } else {
            List ejbIdsList = getEjbIdsThirdPartySPI(ReplicationUtil.getInstanceName());
            stillOwnedEjbIdsForReplicaCacheReconciliation
                = RollingUpgradeUtil.mergeQueryListIntoMap(ejbIdsList, stillOwnedEjbIdsForReplicaCacheReconciliation);
        }
        /*
        Iterator it = stillOwnedEjbIdsForReplicaCacheReconciliation.values().iterator();
        while(it.hasNext()) {
            System.out.println("next id = " + ((FederatedQueryListElement)it.next()).getId());
        }
         */
        reconcileEjbReplicaCache(stillOwnedEjbIdsForReplicaCacheReconciliation);
        //System.out.println("end doReplicaCacheReconciliation");
    }

    public ConcurrentHashMap<String, FederatedQueryListElement> triggerGetIdsForReplicaCacheReconciliation(long waitTime) {
        long startTime = 0L;
        if (_logger.isLoggable(Level.FINE)) {
            startTime = System.currentTimeMillis();
        }
        //because sending to all instances
        int numberOfRespondants = ReplicationUtil.getNumberExpectedRespondants();
        getIdsForReplicaCacheReconciliationDoneSignal = new CountDownLatch(numberOfRespondants);
        //trigger replicateTo partner(s) to
        //provide owned session ids list(s)
        ownedIdsForReplicaCacheReconciliation
            = new ConcurrentHashMap<String, ReplicationState>();
        doTriggerGetIdsForReplicaCacheReconciliation(ReplicationUtil.checkIsInstanceLoadBalancedByCLB());
        ConcurrentHashMap<String, FederatedQueryListElement> result = null;
        try {
            getIdsForReplicaCacheReconciliationDoneSignal.await(waitTime, TimeUnit.SECONDS);
            result = processCacheReconciliationResults(ownedIdsForReplicaCacheReconciliation);
        } catch(InterruptedException ex) {
            result = new ConcurrentHashMap<String, FederatedQueryListElement>();
        } finally {
            if(getIdsForReplicaCacheReconciliationDoneSignal.getCount() != 0) {
                _logger.log(Level.WARNING, "ReplicationManagerBase>>triggerGetIdsForReplicaCacheReconciliation timed out after "
                    + waitTime + " seconds");
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationManagerBase triggerGetIdsForReplicaCacheReconciliation successful after a wait: wait time = " + (System.currentTimeMillis() - startTime));
                }
            }
            ownedIdsForReplicaCacheReconciliation = null;
        }
        return result;
    }

    protected void doTriggerGetIdsForReplicaCacheReconciliation(boolean isInstanceLoadBalancedByCLB) {
        //trigger all partners to do
        //replica cache reconciliation
        sendRollingUpgradeGetIdsForReplicaCacheReconciliationAdvisory();
    }
   
    void cleanOutZombieReplicas() {
        //FIXME for now issue load acknowledgement for each
        //active cache entry
        //this is functionally correct but not optimal
        //solution will be to improve query for replica
        //ids to track the source instance and then only
        //issue load acks to those ids that are not from
        //our replicate to partner
        //FIXME
    }


    public void processRollingupgradegetidsforreplicacachereconciliationadvisory(ReplicationState replicationState) {
        String instanceName = ReplicationUtil.getInstanceName();
        String owningInstanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processRollingupgradegetidsforreplicacachereconciliationadvisory:instance: " + instanceName);
            _logger.fine("in " + this.getClass().getName() + ">>processRollingupgradegetidsforreplicacachereconciliationadvisory:owningInstance=" + owningInstanceName);
        }
        List<FederatedQueryListElement> sessionIds = getEjbIds(owningInstanceName);
        if(this.isThirdPartyBackingStoreInUse()) {
            sessionIds = getEjbIdsThirdPartySPI(owningInstanceName);
        }
        sendReplicasList(sessionIds, owningInstanceName, MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_RESPONSE);
    }

    public void processRollingupgradegetidsforreplicacachereconciliationresponse(ReplicationState replicationState) {
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradegetidsforreplicacachereconciliationresponse received from " + instanceName);
        }
        ownedIdsForReplicaCacheReconciliation.putIfAbsent(instanceName, replicationState);
        getIdsForReplicaCacheReconciliationDoneSignal.countDown();
    }
   
    /**
     * reconcile the replica cache (of your replica partner)
     * with a map containing
     * current set of instance ids owned by this instance
     * @param stillOwnedIds
     */   
    private void reconcileEjbReplicaCache(ConcurrentHashMap stillOwnedIds) {
        //todo
        //FIXME
    }
   
    protected void sendLoadAcknowledgement(String id, long version) {
        JxtaReplicationSender sender
            = JxtaReplicationSender.createInstance();
        ReplicationState loadReceivedState =
            ReplicationState.createBroadcastLoadReceivedState(MODE_EJB, id, this.getApplicationId(), version, ReplicationUtil.getInstanceName(), MESSAGE_BROADCAST_LOAD_RECEIVED);
        ReplicationHealthChecker healthChecker = ReplicationHealthChecker.getInstance();
        String currentReplicaPartner = healthChecker.getCurrentPartnerInstanceName();
            loadReceivedState.setProperty(ReplicationState.IGNORE_REMOVE_INSTANCE_NAME,
            currentReplicaPartner);        
        sender.sendBroadcastQuery(loadReceivedState);
    }   

    List<FederatedQueryListElement> getEjbIds(String owningInstanceName) {
        List<FederatedQueryListElement> ejbIds = new ArrayList();
        //using set to avoid dups
        HashSet ejbIdsSet = new HashSet();
        List replicasToRemove = new ArrayList();
        ReplicationHealthChecker healthChecker = ReplicationHealthChecker.getInstance();
        String correctBuddyReplicaName
            = healthChecker.getReshapeReplicateToInstanceName(null, owningInstanceName, 0L);
        //iterate over ejb replicas
        Iterator it = getReplicatedSessions().values();
        while(it.hasNext()) {
            ReplicationState nextState
                = (ReplicationState)it.next();
            if(!RollingUpgradeUtil.filterOwnership(owningInstanceName, correctBuddyReplicaName, nextState, ejbIdsSet)) {
                //we remove this entry if it is not on the correct replica partner - i.e. it is a zombie
                replicasToRemove.add(nextState.getId());
            }
        }
        for(int i=0; i<replicasToRemove.size(); i++) {
            String nextId = (String)replicasToRemove.get(i);
            if(nextId != null) {
                removeFromReplicationCache(nextId);
            }
        }       
        ejbIds.addAll(ejbIdsSet);
        return ejbIds;
    }

    /**
     * process the broadcastfindejbids for SFSB
     * @param queryState
     */
    public ReplicationState processBroadcastfindejbids(ReplicationState queryState) {
        //complete query and send back response
        String instanceName = ReplicationUtil.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindejbids:instance: " + instanceName);
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindejbids:owningInstance=" + queryState.getInstanceName());
        }
        String owningInstanceName = queryState.getInstanceName();
        List ejbIds = null;
        if(this.isThirdPartyBackingStoreInUse()) {
            ejbIds
                = getEjbIdsThirdPartySPI(owningInstanceName);
        } else {
            ejbIds
                = getEjbIds(owningInstanceName);
        }
        byte[] resultState = null;
        try {
            resultState = ReplicationUtil.getByteArray((ArrayList)ejbIds);
        } catch (IOException ex) {
            //deliberate no-op
            ;
        }
        //first check for none found in either active or replica caches
        if(ejbIds == null || ejbIds.isEmpty()) {
            //return nack
            return ReplicationState.createQueryResponseFrom(queryState, true);
        } else {
            return ReplicationState.createQueryResponseFrom(queryState, resultState);
        }
    }

    List<FederatedQueryListElement> getEjbIdsThirdPartySPI(String owningInstanceName) {
        /* todo fix this
        BackingStore backingStore
            = this.getBackingStore();
        HttpSessionExtraParams httpExtraParamCriteria
            = HttpSessionExtraParams.createSearchCriteria(owningInstanceName);
        Collection<HttpSessionExtraParams> httpColl
            = backingStore.findByCriteria(httpExtraParamCriteria);
        */
        List<FederatedQueryListElement> ejbIds = new ArrayList();
        /*
        //using set to avoid dups
        HashSet sessionIdsSet = new HashSet();
        Iterator<HttpSessionExtraParams> httpResults =
            (Iterator<HttpSessionExtraParams>) httpColl.iterator();
        while (httpResults.hasNext()) {
            HttpSessionExtraParams eParam = httpResults.next();
            if (owningInstanceName.equals(eParam.getCurrentOwnerInstanceName())) {
                sessionIdsSet.add(new FederatedQueryListElement((String)eParam.getId(), -1L, ReplicationUtil.getInstanceName()));
            }
        }
        ejbIds.addAll(sessionIdsSet);
        */
        return ejbIds;
    }

    /**
     * send a rolling upgrade advisory message to instance to trigger
     * it to do rolling upgrade reconciliation for the sending
     * instance
     *
     * @param waitTime the waitTime
     * @param instanceName the instance to be sent the rolling upgrade advisory
     */
    public void sendRollingUpgradeAdvisory(long waitTime, String instanceName) {

        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicatSFSBStoreManager",
                             "sendRollingUpgradeAdvisory",
                             new Object[] {instanceName});
        }
        try {
            RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_EJB, MESSAGE_BROADCAST_ROLLING_UPGRADE_ADVISORY, instanceName, waitTime);
        } catch (IOException ex) {}
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicatSFSBStoreManager",
                            "sendRollingUpgradeAdvisory");
        }
    }

    /**
     * broadcast a rolling upgrade advisory message to trigger
     * rolling upgrade reconciliation for the sending
     * instance
     * @param waitTime the waitTime
     */
    protected void sendRollingUpgradeAdvisory(long waitTime) {
        //broadcast rolling upgrade advisory
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_EJB, MESSAGE_BROADCAST_ROLLING_UPGRADE_ADVISORY, waitTime);
    }

    /**
     * send a rolling upgrade advisory message to instance to trigger
     * it to do rolling upgrade active cache restoration for the sending
     * instance
     *
     * @param instanceName the instance to be sent the rolling upgrade advisory
     */
    public void sendRollingUpgradeActiveCacheRestorationAdvisory(String instanceName) {

        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicatSFSBStoreManager",
                             "sendRollingUpgradeActiveCacheRestorationAdvisory",
                             new Object[] {instanceName});
        }
        try {
            RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_EJB, MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_ADVISORY, instanceName);
        } catch (IOException ex) {}
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicatSFSBStoreManager",
                            "sendRollingUpgradeActiveCacheRestorationAdvisory");
        }
    }

    /**
     * broadcast a rolling upgrade active cache restoration advisory message to trigger
     * rolling upgrade active cache restoration for the sending
     * instance
     */
    protected void sendRollingUpgradeActiveCacheRestorationAdvisory() {
        //broadcast rolling upgrade active cache restoration advisory
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_EJB, MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_ADVISORY);
    }

    /**
     * send a rolling upgrade reconciliation complete advisory message to instance to signal
     * to the original caller instance that replica reconciliation is complete
     *
     * @param instanceName the instance to be sent the rolling upgrade reconciliation complete advisory
     */
    public void sendRollingUpgradeReplicaReconciliationCompleteAdvisory(String instanceName) {

        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicatSFSBStoreManager",
                             "sendRollingUpgradeReplicaReconciliationCompleteAdvisory",
                             new Object[] {instanceName});
        }
        try {
            RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_EJB, MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_RECONCILIATION_COMPLETE_ADVISORY, instanceName);
        } catch (IOException ex) {}
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicatSFSBStoreManager",
                            "sendRollingUpgradeReplicaReconciliationCompleteAdvisory");
        }
    }

    /**
     * broadcast a rolling upgrade getIds
     * for replica cache reconciliation advisory message
     *
     */
    protected void sendRollingUpgradeGetIdsForReplicaCacheReconciliationAdvisory() {
        //broadcast rolling upgrade getIds for replica cache reconciliation advisory
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_EJB, MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_ADVISORY);
    }
   
    public void processRollingupgradeadvisory(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradeadvisory received from " + instanceName);
        }
        long waitTime = ((Long)replicationState.getProperty(ReplicationState.WAIT_TIME)).longValue();
        //we have been triggered to do replica
        //cache reconciliation for our replicateTo partner
        doReplicaCacheReconciliation(waitTime, null);// TODO :: check last param, this whole method is unused -- remove it later.
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradeadvisory sending reconciliation completed advisory to: " + instanceName);
        }
        sendRollingUpgradeReplicaReconciliationCompleteAdvisory(instanceName);
    }

    public void processRollingupgradeactivecacherestorationadvisory(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradeactivecacherestorationadvisory received from " + instanceName);
        }
        List<ReplicationState> ownedReplicas = getSessionReplicas(instanceName);
        sendReplicationStateList(ownedReplicas, instanceName);

    }

    public void processRollingupgradeactivecacherestorationresponse(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradeactivecacherestorationresponse received from " + instanceName);
        }
        ownedReplicaListsReceivedForActiveCacheRestoration.putIfAbsent(replicationState.getInstanceName(), replicationState);
        restoreActiveCacheDoneSignal.countDown();
    }

    public void processRollingupgradereplicareconciliationcompleteadvisory(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradereplicareconciliationcompleteadvisory received from " + replicationState.getInstanceName());
        }
        reconcileReplicaCacheDoneSignal.countDown();
    }

    List<ReplicationState> getSessionReplicas(String owningInstanceName) {
        List<ReplicationState> sessionIds = new ArrayList();
        //using set to avoid dups
        HashSet sessionIdsSet = new HashSet();
        //iterate over http replicas
        Iterator it = getReplicatedSessions().values();
        while(it.hasNext()) {
            ReplicationState nextState
                = (ReplicationState)it.next();
            RollingUpgradeUtil.filterOwnershipOfReplicas(owningInstanceName, nextState, sessionIdsSet);
        }
        sessionIds.addAll(sessionIdsSet);
        return sessionIds;
    }

    private void sendReplicationStateList(List<ReplicationState> ownedReplicas,
                               String instanceName) {
        byte[] ownedReplicasState = null;
        try {
            ownedReplicasState = ReplicationUtil.getByteArray((Serializable)ownedReplicas);
        } catch (IOException ex) {}
        //need a unique pseudo-id for this query
        String appId = getApplicationId();
        String id = appId + unique.getAndIncrement();
        ReplicationState ownedReplicasListState = new ReplicationState(MODE_EJB,
                id, appId, 0L, 0L, 0L, null, null, ReplicationUtil.getInstanceName(),
                MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_RESPONSE, ownedReplicasState, null, null);
        if (ownedReplicas == null || ownedReplicas.isEmpty()) {
            ownedReplicasListState.setNack(true);
        }
        JxtaSocketChannel channel = JxtaSocketChannel.getInstance();
        channel.send(ownedReplicasListState, instanceName);
    }


    private void sendReplicasList(List<FederatedQueryListElement> ownedReplicas,
                               String instanceName, String command) {
        byte[] ownedReplicasState = null;
        try {
            ownedReplicasState = ReplicationUtil.getByteArray((Serializable)ownedReplicas);
        } catch (IOException ex) {}
        //need a unique pseudo-id for this query
        String appId = getApplicationId();
        String id = appId + unique.getAndIncrement();
        ReplicationState ownedReplicasListState = new ReplicationState(MODE_EJB,
                id, appId, 0L, 0L, 0L, null, null, ReplicationUtil.getInstanceName(),
                command, ownedReplicasState, null, null);
        if (ownedReplicas == null || ownedReplicas.isEmpty()) {
            ownedReplicasListState.setNack(true);
        }
        JxtaSocketChannel channel = JxtaSocketChannel.getInstance();
        channel.send(ownedReplicasListState, instanceName);
    }
   
    // end rolling upgrade related code    
   
    /**
    * append the debug monitor statistics to the buffer
    */    
    public void appendStats(StringBuffer sb) {
        EJBModuleStatistics stats = this.getEJBModuleStatistics();
        sb.append("\nSAVE_LOW=" + stats.getSaveLow());
        sb.append("\nSAVE_HIGH=" + stats.getSaveHigh());
        sb.append("\nSAVE_AVG=" + stats.getSaveAverage());
       
        sb.append("\nPASSIVATE_SAVE_LOW=" + stats.getPassivateSaveLow());
        sb.append("\nPASSIVATE_SAVE_HIGH=" + stats.getPassivateSaveHigh());
        sb.append("\nPASSIVATE_SAVE_AVG=" + stats.getPassivateSaveAverage());
       
        sb.append("\nCHECKPOINT_SAVE_LOW=" + stats.getCheckpointSaveLow());
        sb.append("\nCHECKPOINT_SAVE_HIGH=" + stats.getCheckpointSaveHigh());
        sb.append("\nCHECKPOINT_SAVE_AVG=" + stats.getCheckpointSaveAverage());             

        sb.append("\nBEAN_SIZE_LOW=" + stats.getBeanSizeLow());
        sb.append("\nBEAN_SIZE_HIGH=" + stats.getBeanSizeHigh());
        sb.append("\nBEAN_SIZE_AVG=" + stats.getBeanSizeAverage());
       
        stats.resetStats();      
    }
   
    //new code start   
   
    /** Removes the bean will be called when the client removes the bean
     * @param id ID of the bean to be removed
     * (see removeNoBatch which removes immediately w/o batching)
     */
    /* FIXME reverting back to non-batch remove for testing
    public void remove(Object id) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicatedSFSBStoreManager", "remove", id);
        }
        if(id == null) {
            return;
        }
        String beanId = id.toString();
        //rLock.lock();
        try {
            removedKeysMap.put(beanId, beanId);
            int requestCount = requestCounter.incrementAndGet();
            if ((( (Math.abs(requestCount)) % NUMBER_OF_REQUESTS_BEFORE_FLUSH) == 0)) {
                boolean wakeupDispatcher = timeToChange.compareAndSet(false, true); //expect false  set  to true
                if (wakeupDispatcher) {
                    dispatchThread.wakeup();
                }
            }
        } finally {
            //rLock.unlock();
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicatedSFSBStoreManager", "remove");
        }       
    }
     */
   
    //Called by the dispatcher
    void flushAllIdsFromCurrentMap(boolean waitForAck) {
        Map oldKeysMap = null;
        try {
            oldKeysMap = removedKeysMap;
            removedKeysMap = new ConcurrentHashMap<String, String>();
            timeToChange.set(false);
        } finally {
        }

        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine(">>ReplicationSFSBStoreManager::flushAllIds: " + oldKeysMap.size());
        }        
        //_logger.log(Level.INFO, ">>ReplicationSFSBStoreManager::flushAllIds: " + oldKeysMap.size());
       
        //Send sessions in currentMap into a message
        List<String> list = new ArrayList<String>(oldKeysMap.size()+1);
        Iterator<String> iter = oldKeysMap.keySet().iterator();
        int totalMessageSize = 0;
        while (iter.hasNext()) {
            String nextString = iter.next();
            list.add(nextString);
        }       
        if (list.size() > 0) {
            createRemoveIdsMessageAndSend(list, waitForAck, null);
        }
    }
   
    private void createRemoveIdsMessageAndSend(List<String> list, boolean waitForAck, Object signalObject) {
        byte[] data = null;
        ByteArrayOutputStream bos = null;
        ObjectOutputStream oos = null;
        try {
            bos = new ByteArrayOutputStream();
            oos = new ObjectOutputStream(bos);
            oos.writeObject(list);
            oos.flush();
        } catch (IOException ioEx) {
            //FIXME
        } finally {
            if (oos != null) {
                try {
                    oos.flush(); oos.close();
                } catch (Exception ex) {
                    //Ignore
                }
            }
           
            if (bos != null) {
                try {
                    bos.flush();
                    data = bos.toByteArray();
                    bos.close();
                } catch (Exception ex) {
                    //FIXME
                }
            }
        }

        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("<<ReplicationWebEventPersistentManager::createRemoveAllMessageAndSend: About to send " + data.length + " bytes...");
        }               
        removeIds(_messageIDCounter++, list.size(), data, signalObject);
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("<<ReplicationWebEventPersistentManager::createRemoveAllMessageAndSend: DONE!!");
        }
    }
   
   /**
   * does the remove of all session ids in removedIdsData
   *
   * @param msgID message id for this remove all message
   * @param removedIdsData serialized list of ids to remove
   */   
    protected void removeIds(long msgID, int totalStates, byte[] removedIdsData, Object signalObject) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicatedSFSBStoreManager", "removeIds");
        }
        ReplicatedEjbStore store = null;
        try {
            store = getStore();
           
            store.removeIds(msgID, removedIdsData);
            if(_logger.isLoggable(Level.FINE)) {
                _logger.exiting("ReplicatedSFSBStoreManager", "removeIds");
            }
           
        }
        catch (Exception e) {
            e.printStackTrace();
        } finally {
            this.putStore(store);
        }       
       
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicatedSFSBStoreManager", "removeIds");
        }
    }
   
    public void processRemoveids(ReplicationState message) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN" + this.getClass().getName() + ">>processRemoveids");
        }
        //state of this message contains serialized list of ids to remove
        byte[] idsToRemoveState = message.getState();
        List removedIdsList = new ArrayList();
        BaseCache replicatedSessionsCache = getReplicatedSessions();
        try {
            removedIdsList = (List)ReplicationState.getObjectValue(idsToRemoveState);
        } catch (Exception ex) {
            //deliberately do nothing
        }
        //ReplicationState.displayStringList(removedIdsList);
        for(int i=0; i<removedIdsList.size(); i++) {
            String nextIdToRemove = (String)removedIdsList.get(i);
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine(">>processRemoveids:nextIdToRemove=" + nextIdToRemove);
            }           
            //_logger.info(">>processRemoveids:nextIdToRemove=" + nextIdToRemove);
            replicatedSessionsCache.remove(nextIdToRemove);
        }
    }   
   
    private static int NUMBER_OF_REQUESTS_BEFORE_FLUSH = 1000;
    volatile Map<String, String> removedKeysMap = new ConcurrentHashMap<String, String>();
    private static AtomicInteger requestCounter = new AtomicInteger(0);
    private static int _messageIDCounter = 0;
    private AtomicBoolean  timeToChange = new AtomicBoolean(false);
    private DispatchThread dispatchThread = new DispatchThread();   
   
    private class DispatchThread implements Runnable {
       
        private volatile boolean done = false;
      
        private Thread thread;
       
        private LinkedBlockingQueue<Object> queue;
       
        public DispatchThread() {
            this.queue = new LinkedBlockingQueue<Object>();
            this.thread = new Thread(this);
            this.thread.setDaemon(true);
            thread.start();
        }
       
        public void wakeup() {
            queue.add(new Object());
        }
       
        public void run() {
            while (! done) {
                try {
                    Object ignorableToken = queue.take();
                    flushAllIdsFromCurrentMap(false);
                } catch (InterruptedException inEx) {
                    this.done = true;
                }
            }
        }

    }
   
    private class CleanupThread implements Runnable {
       
        private volatile boolean threadDone = false;
       
        private long sleepIntervalSeconds = 60L;
      
        private Thread thread;
       
        public CleanupThread() {
            this.thread = new Thread(this);
            this.thread.setDaemon(true);
            thread.start();
        }
       
        public void run() {
            while (!threadDone) {               
                try {
                    Thread.sleep(sleepIntervalSeconds * 1000L);           
                } catch (InterruptedException e) {
                    ;
                }
                backgroundProcess();
            }
        }
       
        /**
         * Stop the background thread
         */
        protected void threadStop() {
            if (this.thread == null)
                return;

            threadDone = true;
            this.thread.interrupt();
            try {
                this.thread.join();
            } catch (InterruptedException e) {
                ;
            }
        }
    }

    private class ReplicationStateUpdate {

        private String _id = null;
        private long _lastAccess = 0L;
        private long _version = -1L;
        private byte[] _containerExtraParams = null;
       
        String getId() {
            return _id;
        }       
       
        long getLastAccess() {
            return _lastAccess;
        }
       
        long getVersion() {
            return _version;
        }
       
        byte[] getContainerExtraParams() {
            return _containerExtraParams;
        }       
       
        public ReplicationStateUpdate(String id, long version, long lastAccess, byte[] containerExtraParams) {
            _id = id;
            _version = version;
            _lastAccess = lastAccess;
            _containerExtraParams = containerExtraParams;
        }
       
    }
}
TOP

Related Classes of com.sun.ejb.ee.sfsb.store.ReplicatedSFSBStoreManager$DispatchThread

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.