Package com.sun.enterprise.ee.web.sessmgmt

Source Code of com.sun.enterprise.ee.web.sessmgmt.ReplicationManagerBase

/*
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
*
* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
*
* The contents of this file are subject to the terms of either the GNU
* General Public License Version 2 only ("GPL") or the Common Development
* and Distribution License("CDDL") (collectively, the "License").  You
* may not use this file except in compliance with the License. You can obtain
* a copy of the License at https://glassfish.dev.java.net/public/CDDL+GPL.html
* or glassfish/bootstrap/legal/LICENSE.txt.  See the License for the specific
* language governing permissions and limitations under the License.
*
* When distributing the software, include this License Header Notice in each
* file and include the License file at glassfish/bootstrap/legal/LICENSE.txt.
* Sun designates this particular file as subject to the "Classpath" exception
* as provided by Sun in the GPL Version 2 section of the License file that
* accompanied this code.  If applicable, add the following below the License
* Header, with the fields enclosed by brackets [] replaced by your own
* identifying information: "Portions Copyrighted [year]
* [name of copyright owner]"
*
* Contributor(s):
*
* If you wish your version of this file to be governed by only the CDDL or
* only the GPL Version 2, indicate your decision by adding "[Contributor]
* elects to include this software in this distribution under the [CDDL or GPL
* Version 2] license."  If you don't indicate a single choice of license, a
* recipient has the option to distribute your version of this file under
* either the CDDL, the GPL Version 2 or to extend the choice of license to
* its licensees as provided above.  However, if you add GPL Version 2 code
* and therefore, elected the GPL Version 2 license, then the option applies
* only if the new code is made subject to such option by the copyright
* holder.
*/
/*
* ReplicationManagerBase.java
*
* Created on October 31, 2006, 10:55 AM
*
*/

package com.sun.enterprise.ee.web.sessmgmt;

import com.sun.appserv.ha.spi.BackingStore;
import com.sun.appserv.ha.spi.BackingStoreException;
import com.sun.appserv.ha.spi.BackingStoreFactory;
import com.sun.appserv.ha.spi.BackingStoreRegistry;
import com.sun.appserv.ha.util.CompositeMetadata;
import com.sun.appserv.ha.util.SessionAttributeMetadata;
import com.sun.appserv.ha.util.SimpleMetadata;
import com.sun.appserv.ha.uow.ReplicableEntity;
import com.sun.appserv.util.cache.BaseCache;
import com.sun.enterprise.ee.web.initialization.ServerConfigReader;
import com.sun.enterprise.web.MonitorStatsCapable;
import com.sun.enterprise.web.ServerConfigLookup;
import com.sun.enterprise.web.ShutdownCleanupCapable;
import com.sun.logging.LogDomains;
import org.apache.catalina.Globals;
import org.apache.catalina.LifecycleException;
import org.apache.catalina.Session;
import org.apache.catalina.session.PersistentManagerBase;
import org.apache.catalina.session.StandardSession;

import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.security.AccessController;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
*
* @author Larry White
*/
public abstract class ReplicationManagerBase extends PersistentManagerBase
        implements MonitorStatsCapable, DynamicOwnershipManager, PurgeCapable {
   
    public final static String LOGGER_MEM_REP
        = ReplicationState.LOGGER_MEM_REP;
   
    private final static Level TRACE_LEVEL = Level.FINE;
   
    final static String DUPLICATE_IDS_SEMANTICS_PROPERTY
        = ReplicationState.DUPLICATE_IDS_SEMANTICS_PROPERTY;
    final static String SUPPRESS_LOAD_ACK_PROPERTY
        = ReplicationState.SUPPRESS_LOAD_ACK_PROPERTY;
    final static String SESSION_MANAGER_PROPERTY =
            ReplicationState.SESSION_MANAGER_PROPERTY; // used during jxtabackingstore creation to store a reference to this session manager in jxtabackingstore.
    public final static String BEKEY
        = ReplicationState.BEKEY;
    private static boolean useReplicationUnicastLoadResponseBatching = false;

    public final static String LOAD_SESSION_COMMAND = "loadsession";
    final static String MESSAGE_BROADCAST_LOAD_RECEIVED
        = ReplicationState.MESSAGE_BROADCAST_LOAD_RECEIVED;   
    public final static String MESSAGE_BROADCAST_EXPAT_QUERY_SESSION
        = "broadcastfindsessionexpatids";
    public final static String MESSAGE_BROADCAST_EXPAT_QUERY_SESSION_REPLICA
        = "broadcastfindsessionreplicaexpatids";
    public final static String MESSAGE_BROADCAST_EXPAT_RECEIVE_SESSION
        = "broadcastreceivesessionexpatids";
    public final static String MESSAGE_BROADCAST_EXPAT_RECEIVE_SESSION_REPLICA
        = "broadcastreceivesessionreplicaexpatids";
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_RESPONSE
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_RESPONSE;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_RESPONSE
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_RESPONSE;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_RECONCILIATION_COMPLETE_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_RECONCILIATION_COMPLETE_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_RESPONSE
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_RESPONSE;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_RESPONSE
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_RESPONSE;

    private final static String EXTENDED_UTILITY_CLASS_NAME
        = "org.jvnet.glassfish.comms.replication.sessmgmt.ExtendedReplicationUtil";
   
    /**
     * The logger to use for logging ALL web container related messages.
     */
    private static final Logger _logger
        = Logger.getLogger(LOGGER_MEM_REP);   
    private static final Logger _pipelogger = ReplicationUtil.getPipeLogger();
    private static final Logger _salogger = ReplicationUtil.getSALogger();
   
    protected static int _maxBaseCacheSize = 4096;
    protected static float _loadFactor = 0.75f;
    protected static final int _concurrencyLevel = 100;
   
    public static final String RECONCILE_HTTPSESSION_REQUEST =
            "reconcilehttpsessionrequest";
    public static final String RECONCILE_HTTPSESSION_RESPONSE =
            "reconcilehttpsessionresponse";
    public static final String RECONCILE_HTTPSESSION_REPLICA_REQUEST =
            "reconcilehttpsessionreplicarequest";
    public static final String RECONCILE_HTTPSESSION_REPLICA_RESPONSE =
            "reconcilehttpsessionreplicaresponse";
    public static final String POST_RECONCILE_HTTPSESSION_REQUEST =
            "postreconcilehttpsessionrequest";
    public static final String POST_RECONCILE_HTTPSESSION_RESPONSE =
            "postreconcilehttpsessionresponse";

    private volatile AtomicReference<CountDownLatch> activeHttpSessionReconcileSignal =
            new AtomicReference<CountDownLatch>();
    private volatile AtomicReference<CountDownLatch> replicaHttpSessionReconcileSignal =
            new AtomicReference<CountDownLatch>();
    private volatile AtomicReference<CountDownLatch> postHttpSessionReconcileSignal =
            new AtomicReference<CountDownLatch>();
    private volatile Map<String, ReplicationState> reconciledActiveHttpSessions =
            new HashMap<String, ReplicationState>();
    private volatile Map<String, ReplicationState> reconciledReplicaHttpSessions =
            new HashMap<String, ReplicationState>();
    private volatile Map<String, ReplicationState> remotelyLockedHttpSessions =
            new HashMap<String, ReplicationState>();

    /**
     * the list of method names that are broadcasts or unicast
     */
    private static List broadcastMethods
        = Arrays.asList(
            MESSAGE_BROADCAST_EXPAT_QUERY_SESSION,
            MESSAGE_BROADCAST_EXPAT_QUERY_SESSION_REPLICA,
            MESSAGE_BROADCAST_EXPAT_RECEIVE_SESSION,
            LOAD_SESSION_COMMAND,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_ADVISORY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_ADVISORY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_RESPONSE,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_ADVISORY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_RESPONSE,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_ADVISORY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_RESPONSE,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_ADVISORY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_RESPONSE,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_RECONCILIATION_COMPLETE_ADVISORY);

    /**
     * the list of method names that are expensive
     * i.e. should not run on single receiving thread
     * e.g. expat list query which when received take some time
     * which should not occur on the same receiving thread
     */
    private static List expensiveMethods
        = Arrays.asList(
            MESSAGE_BROADCAST_EXPAT_QUERY_SESSION,
            MESSAGE_BROADCAST_EXPAT_QUERY_SESSION_REPLICA);

    /**
     * the list of method names that are removes
     */
    private static List removeMethods
        = Arrays.asList(ReplicationState.REMOVE_COMMAND,
            ReplicationState.REMOVE_SYNCHRONIZED_COMMAND);
    private static List saveMethods
  = Arrays.asList(ReplicationState.SAVE_COMMAND);
  
    static
  {
            checkSessionCacheProperties();
            registerBroadcastMethods();
            registerExpensiveMethods();
            registerRemoveMethods();
            registerSaveMethods();
            ServerConfigLookup lookup = new ServerConfigLookup();
            useReplicationUnicastLoadResponseBatching
                = lookup.isReplicationUnicastLoadResponseBatchingEnabled();           
  }

    private static ReplicationMessageRouter getRouter() {
        ReplicationMessageRouter receiver = null;
        if (Globals.IS_SECURITY_ENABLED) {
            receiver = (ReplicationMessageRouter)
                AccessController.doPrivileged(
                    new PrivilegedGetReplicationMessageRouter());
        } else {
            receiver = ReplicationMessageRouter.createInstance();
        }
        return receiver;
    }

    protected static void registerBroadcastMethods() {
        ReplicationMessageRouter router = getRouter();
        router.registerBroadcastMethodList(broadcastMethods);
    }

    protected static void registerExpensiveMethods() {
        ReplicationMessageRouter router = getRouter();
        router.registerExpensiveMethodList(expensiveMethods);
    }

    protected static void registerRemoveMethods() {
        ReplicationMessageRouter router = getRouter();
        router.registerRemoveMethodList(removeMethods);
    }

    protected static void registerSaveMethods() {
        ReplicationMessageRouter router = getRouter();
        router.registerSaveMethodList(saveMethods);
    }

    protected static boolean checkSessionCacheProperties() {
        boolean result = false;
  try
        {
            Properties props = System.getProperties();
            String cacheSize=props.getProperty("HTTP_SESSION_CACHE_MAX_BASE_CACHE_SIZE");
            if(null!=cacheSize) {
                _maxBaseCacheSize = (new Integer (cacheSize).intValue());
           
            String loadFactor=props.getProperty("HTTP_SESSION_CACHE_MAX_BASE_LOAD_FACTOR");
            if(null!=loadFactor) {
                _loadFactor = (new Float (loadFactor).floatValue());
            }
        } catch(Exception e)
        {
            //do nothing accept defaults
        }
        return result;
    }   
   
    protected final static String MODE_WEB = ReplicationState.MODE_WEB;

    protected static java.util.concurrent.atomic.AtomicLong unique =
        new AtomicLong(0);

    private ReplicationUtil replicationUtil = ReplicationUtil.createReplicationUtil();
   
    private ReentrantReadWriteLock expatReadWriteLock
            = new ReentrantReadWriteLock();

    private HashMap<String,String> lastExpatQueryId
        = new HashMap<String,String>();

    protected ConcurrentHashMap<String, ReplicationState> ownedReplicaListsReceivedForActiveCacheRestoration = null;
    protected ConcurrentHashMap<String, ReplicationState> ownedReplicaListsReceivedForReplicaCacheRestoration = null;
    protected ConcurrentHashMap<String, ReplicationState> ownedIdsForReplicaCacheReconciliation = null;
    protected ConcurrentHashMap<String, ReplicationState> ownedIdsForActiveCacheReconciliation = null;

    private volatile CountDownLatch reconcileReplicaCacheDoneSignal = null;
    private volatile CountDownLatch getIdsForReplicaCacheReconciliationDoneSignal = null;
    private volatile CountDownLatch getIdsForActiveCacheReconciliationDoneSignal = null;
    private volatile CountDownLatch restoreActiveCacheDoneSignal = null;
    private volatile CountDownLatch restoreReplicaCacheDoneSignal = null;
   
    ExpatListHandler sessExpatListHandler;
    ExpatListHandler sessReplicaExpatHandler;
   
    /** Creates a new instance of ReplicationManagerBase */
    public ReplicationManagerBase() {
        super();
        replicatedSessionMonitors = new ReplicationSessionMonitors(_salogger, _maxBaseCacheSize, _loadFactor);
        ServerConfigLookup lookup = new ServerConfigLookup();
        replicationCompressionEnabled = lookup.isReplicationCompression();
    }
   
    private void getBackingStores() {
        // create backing stores so that the replica expat handlers get created
        // well in advance. Otherwise the expat pushed by remote instances
        // might get missed out. This can happen in the restarted instance.
        try {
            getBackingStore();
        } catch (Exception ex) {
            ex.printStackTrace();
        }
    }

    public void start() throws LifecycleException {
        super.start();
        sessExpatListHandler = ExpatListHandler.getExpatListHandler(this, this.getApplicationId(),
            MESSAGE_BROADCAST_EXPAT_QUERY_SESSION, MODE_WEB, "HttpSessionActive");
        getBackingStores();
        registerWithMessageRouter();
    }
   
    /**
     * Save all currently active sessions in the appropriate persistence
     * mechanism, if any.  If persistence is not supported, this method
     * returns without doing anything.
     * <p>
     * Note that by default, this method is not called by the MiddleManager
     * class. In order to use it, a subclass must specifically call it,
     * for example in the stop() and/or processPersistenceChecks() methods.
     */
    @Override
    public void unload() {
        //deliberate no-op we do not want to unload this way
        //because we have an alternate configurable method
        //driven from ReplicationLifeycleImpl
    }   
   
    protected void registerWithMessageRouter() {
        ReplicationMessageRouter router = getRouter();
        if(router != null) {
            router.addReplicationManager(this.getApplicationId(), (ReplicationManager)this);
        }
    }

    ExpatListElement getExpatListElement(String id) {
        if(expatReadWriteLock.readLock().tryLock()) {
            try {
    synchronized(expatIdsMap) {
                    ExpatListElement result = expatIdsMap.get(id);
                    return result;
    }
            } finally {
                expatReadWriteLock.readLock().unlock();
            }
        } else {
            return null;
        }
    }

    ExpatListElement removeExpatListElement(String id) {
        if(expatReadWriteLock.readLock().tryLock()) {
            try {
    synchronized(expatIdsMap) {
                    ExpatListElement result = expatIdsMap.remove(id);
                    return result;
    }
            } finally {
                expatReadWriteLock.readLock().unlock();
            }
        } else {
            return null;
        }
    }
   
    // begin post join reconciliation

    /**
     * do reconciliation processing
     * used for both rolling upgrade and post network partition rejoin
     * @param waitTime the waitTime in seconds
     * @param ctx the RollingUpgradeContext
     */
    public void doPostJoinReconciliation(long waitTime, RollingUpgradeContext ctx) {
        long startTime = 0L;
        if (_logger.isLoggable(Level.FINE)) {
            startTime = System.currentTimeMillis();
        }
        CountDownLatch doneSignal = new CountDownLatch(2);
        //do active cache reconciliation
        ReconcileActive reconcileActive
            = new ReconcileActive((ReplicationManager)this, waitTime, doneSignal, ctx);
        RollingUpgradeHandler.executeTask(reconcileActive);

        //trigger replica cache reconciliation
        TriggerReconcileReplica triggerReconcileReplica
            = new TriggerReconcileReplica((ReplicationManager)this, waitTime, doneSignal, ctx);
        RollingUpgradeHandler.executeTask(triggerReconcileReplica);

        try {
            doneSignal.await(waitTime, TimeUnit.SECONDS);
        } catch(InterruptedException ex) {
            ;
        } finally {
            if(doneSignal.getCount() != 0) {
                String errorMsg = "ReplicationManagerBase>>doPostJoinReconciliation timed out after " + waitTime + " seconds";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationManagerBase doPostJoinReconciliation successful after a wait: wait time = " + (System.currentTimeMillis() - startTime));
                }
            }
        }
    }

    // end post join reconciliation
   
    // begin rolling upgrade related code
   
    public void doRollingUpgradePreShutdownProcessing(long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        //do syncpoint save
        doSyncpointSave(waitTime, doneSignal, ctx);
    }

    public void doRollingUpgradePostStartupProcessing(String rollingUpgradeType, long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        if(RollingUpgradeHandler.isFileBased(rollingUpgradeType)) {
            //do syncpoint load
            doSyncpointLoad(waitTime, doneSignal, ctx);
        } else {
            doMemoryLoad(waitTime, doneSignal, ctx);
        }
    }

    public void doRollingUpgradePostStartupReconciliationProcessing(long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        //do post join reconciliation
        try {
            doPostJoinReconciliation(waitTime, ctx);
        } finally {
            doneSignal.countDown();
        }
    }
   
    void doSyncpointSave(long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        try {
            if(this.isSkipRollingUpgradeBackupRestore()) {
                return;
            }
            WebFileSync syncStore = new WebFileSync((ReplicationManager)this);
            try {
                syncStore.save(waitTime, ctx);
            } catch (IOException ex) {
                ex.printStackTrace();
                String errorMsg = "ReplicationManagerBase>>doSyncPointSave IOException";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            }
        } finally {
            doneSignal.countDown();
        }
    }

    void doSyncpointLoad(long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        try {
            if(this.isSkipRollingUpgradeBackupRestore()) {
                return;
            }
            WebFileSync syncStore = new WebFileSync((ReplicationManager)this);
            try {
                syncStore.load(waitTime, ctx);
                markActiveCacheAsSuspected(true);
            } catch (IOException ex) {
                String errorMsg = "ReplicationManagerBase>>doSyncPointLoad IOException";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } catch (ClassNotFoundException ex2) {
                String errorMsg = "ReplicationManagerBase>>doSyncPointLoad ClassNotFoundException";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);               
            }
        } finally {
            doneSignal.countDown();
        }
    }

    void doMemoryLoad(long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        try {
            if(this.isSkipRollingUpgradeBackupRestore()) {
                return;
            }
            WebMemorySync syncStore = new WebMemorySync((ReplicationManager)this);
            try {
                syncStore.load(waitTime, ctx);
                markActiveCacheAsSuspected(true);
            } catch (IOException ex) {
                String errorMsg = "ReplicationManagerBase>>doMemoryLoad IOException";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } catch (ClassNotFoundException ex2) {
                String errorMsg = "ReplicationManagerBase>>doMemoryLoad ClassNotFoundException";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            }
        } finally {
            doneSignal.countDown();
        }
    }

    void restoreActiveCacheViaMemory(long waitTime, RollingUpgradeContext ctx) {
        triggerActiveCacheRestoration(waitTime, ctx);
    }

    void restoreReplicaCacheViaMemory(long waitTime, RollingUpgradeContext ctx) {
        triggerReplicaCacheRestoration(waitTime, ctx);
    }

    public void triggerActiveCacheRestoration(long waitTime, RollingUpgradeContext ctx) {
        long startTime =  System.currentTimeMillis();
        int numberOfRespondants = RollingUpgradeUtil.getNumberOfExpectedRespondants();
        restoreActiveCacheDoneSignal = new CountDownLatch(numberOfRespondants);
        //trigger replicateTo partner(s) to do
        //active cache restoration
        ownedReplicaListsReceivedForActiveCacheRestoration
            = new ConcurrentHashMap<String, ReplicationState>();
        doTriggerActiveCacheRestoration(ReplicationUtil.checkIsInstanceLoadBalancedByCLB());

        try {
            restoreActiveCacheDoneSignal.await(waitTime, TimeUnit.SECONDS);
            processActiveCacheRestorationResults(ownedReplicaListsReceivedForActiveCacheRestoration);
        } catch(InterruptedException ex) {
            ;
        } finally {
            if(restoreActiveCacheDoneSignal.getCount() != 0) {
                String errorMsg = "ReplicationManagerBase>>triggerActiveCacheRestoration timed out after " + waitTime + " seconds";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationManagerBase triggerActiveCacheRestoration successful after a wait: wait time = " + (System.currentTimeMillis() - startTime) + " active HttpSessions restored: " + sessions.size());
                }
            }
            ctx.addMessage(getApplicationId(), "Active cache restoration completed for HttpSession. " +
                    ", Size of active cache = " + sessions.size() +
                    ", Time taken (ms) = " + (System.currentTimeMillis()-startTime));

            ownedReplicaListsReceivedForActiveCacheRestoration = null;
        }
    }

    protected void doTriggerActiveCacheRestoration(boolean isInstanceLoadBalancedByCLB) {
        if(!isInstanceLoadBalancedByCLB) {
            //trigger replicateTo partner to do
            //active cache restoration
            ReplicationHealthChecker healthChecker
                = ReplicationHealthChecker.getInstance();
            String replicateToInstanceName = healthChecker.getReshapeReplicateToInstanceName(null, 0L);
            sendRollingUpgradeActiveCacheRestorationAdvisory(replicateToInstanceName);
        } else {
            //trigger all partners to do
            //active cache restoration
            sendRollingUpgradeActiveCacheRestorationAdvisory();
        }
    }

    protected void processActiveCacheRestorationResults(ConcurrentHashMap<String, ReplicationState> ownedReplicaListsReceivedForActiveCacheRestoration) {
        ReplicationStore store = getSingletonStore();
        Collection<ReplicationState> ownedReplicasList = ownedReplicaListsReceivedForActiveCacheRestoration.values();
        Iterator<ReplicationState> it = ownedReplicasList.iterator();
        while(it.hasNext()) {
            ReplicationState nextOwnedListReplicationState = it.next();
            List<ReplicationState>nextOwnedListOfStates
                = (List<ReplicationState>)RollingUpgradeUtil.getObject(nextOwnedListReplicationState.getState());
            for(int i=0; i<nextOwnedListOfStates.size(); i++) {
                ReplicationState nextState = nextOwnedListOfStates.get(i);
                Session nextSession = null;
                try {
                    nextSession = store.getSession(nextState);
                } catch (Exception ex) {}
                if(nextSession != null) {
                    this.add(nextSession);
                }
            }
        }
    }

    public void triggerReplicaCacheRestoration(long waitTime, RollingUpgradeContext ctx) {
        long startTime = System.currentTimeMillis();
        int numberOfRespondants = RollingUpgradeUtil.getNumberOfExpectedRespondants();
        restoreReplicaCacheDoneSignal = new CountDownLatch(numberOfRespondants);
        //trigger replicateFrom partner(s) to do
        //replica cache restoration
        ownedReplicaListsReceivedForReplicaCacheRestoration
            = new ConcurrentHashMap<String, ReplicationState>();
        doTriggerReplicaCacheRestoration(ReplicationUtil.checkIsInstanceLoadBalancedByCLB());

        try {
            restoreReplicaCacheDoneSignal.await(waitTime, TimeUnit.SECONDS);
            processReplicaCacheRestorationResults(ownedReplicaListsReceivedForReplicaCacheRestoration);
        } catch(InterruptedException ex) {
            ;
        } finally {
            if(restoreReplicaCacheDoneSignal.getCount() != 0) {
                String errorMsg = "ReplicationManagerBase>>triggerReplicaCacheRestoration timed out after " + waitTime + " seconds";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationManagerBase triggerReplicaCacheRestoration successful after a wait: wait time = " + (System.currentTimeMillis() - startTime) + " replica HttpSessions restored: " + replicaCache.getReplicatedSessions().getEntryCount());
                }
            }
            ownedReplicaListsReceivedForReplicaCacheRestoration = null;
            ctx.addMessage(getApplicationId(), "Replica cache restoration completed for HttpSession. " +
                    ", Size of replica cache = " + getReplicatedSessions().getEntryCount() +
                    ", Time taken (ms) = " + (System.currentTimeMillis()-startTime));
        }
    }

    protected void doTriggerReplicaCacheRestoration(boolean isInstanceLoadBalancedByCLB) {
        if(!isInstanceLoadBalancedByCLB) {
            //trigger replicateFrom partner to do
            //replica cache restoration
            ReplicationHealthChecker healthChecker
                = ReplicationHealthChecker.getInstance();
            String replicateFromInstanceName = healthChecker.getReshapeReplicateFromInstanceName(null);
            sendRollingUpgradeReplicaCacheRestorationAdvisory(replicateFromInstanceName);
        } else {
            //trigger all partners to do
            //replica cache restoration
            sendRollingUpgradeReplicaCacheRestorationAdvisory();
        }
    }

    protected void processReplicaCacheRestorationResults(ConcurrentHashMap<String, ReplicationState> ownedReplicaListsReceivedForActiveCacheRestoration) {
        Collection<ReplicationState> ownedReplicasList = ownedReplicaListsReceivedForActiveCacheRestoration.values();
        Iterator<ReplicationState> it = ownedReplicasList.iterator();
        while(it.hasNext()) {
            ReplicationState nextOwnedListReplicationState = it.next();
            List<ReplicationState>nextOwnedListOfStates
                = (List<ReplicationState>)RollingUpgradeUtil.getObject(nextOwnedListReplicationState.getState());
            for(int i=0; i<nextOwnedListOfStates.size(); i++) {
                ReplicationState nextState = nextOwnedListOfStates.get(i);                                
                this.putInReplicationCache(nextState);
            }
        }
    }

    void markActiveCacheAsSuspected(boolean value) {
        //iterate over active sessions cache
        //take copy asap to avoid synchronizing too long
        List sessionsCopiedList = getClonedSessionsList();
        Iterator it = sessionsCopiedList.iterator();
        while(it.hasNext()) {
            ((BaseHASession)it.next()).setSuspect(value);
        }
    }

    public void doActiveCacheReconciliation(long waitTime, RollingUpgradeContext ctx) {
        reconcileSessionActiveCache(waitTime, ctx);
    }

    public ConcurrentHashMap<String, FederatedQueryListElement> triggerGetIdsForActiveCacheReconciliation(long waitTime) {
        long startTime = 0L;
        if (_logger.isLoggable(Level.FINE)) {
            startTime = System.currentTimeMillis();
        }
        //sending to all for both types of load balancers
        int numberOfRespondants = ReplicationUtil.getNumberExpectedRespondants();
        getIdsForActiveCacheReconciliationDoneSignal = new CountDownLatch(numberOfRespondants);
        //trigger replicateTo partner(s) to
        //provide owned session ids list(s)
        ownedIdsForActiveCacheReconciliation
            = new ConcurrentHashMap<String, ReplicationState>();
        doTriggerGetIdsForActiveCacheReconciliation(ReplicationUtil.checkIsInstanceLoadBalancedByCLB());
        ConcurrentHashMap<String, FederatedQueryListElement> result = null;
        try {
            getIdsForActiveCacheReconciliationDoneSignal.await(waitTime, TimeUnit.SECONDS);
            result = processCacheReconciliationResults(ownedIdsForActiveCacheReconciliation);
        } catch(InterruptedException ex) {
            result = new ConcurrentHashMap<String, FederatedQueryListElement>();
        } finally {
            if(getIdsForActiveCacheReconciliationDoneSignal.getCount() != 0) {
                _logger.log(Level.WARNING, "ReplicationManagerBase>>triggerGetIdsForActiveCacheReconciliation timed out after "
                    + waitTime + " seconds");
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationManagerBase triggerGetIdsForActiveCacheReconciliation successful after a wait: wait time = " + (System.currentTimeMillis() - startTime));
                }
            }
            ownedIdsForActiveCacheReconciliation = null;
        }
        return result;
    }

    protected void doTriggerGetIdsForActiveCacheReconciliation(boolean isInstanceLoadBalancedByCLB) {
        //trigger all partners to do
        //active cache reconciliation
        sendRollingUpgradeGetIdsForActiveCacheReconciliationAdvisory();
    }

    /**
     * broadcast a rolling upgrade getIds
     * for replica cache reconciliation advisory message
     *
     */
    protected void sendRollingUpgradeGetIdsForActiveCacheReconciliationAdvisory() {
        //broadcast rolling upgrade getIds for replica cache reconciliation advisory
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_WEB, MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_ADVISORY);
    }

    public void processRollingupgradegetidsforactivecachereconciliationadvisory(ReplicationState replicationState) {
        String instanceName = ReplicationUtil.getInstanceName();
        String owningInstanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processRollingupgradegetidsforactivecachereconciliationadvisory:instance: " + instanceName);
            _logger.fine("in " + this.getClass().getName() + ">>processRollingupgradegetidsforactivecachereconciliationadvisory:owningInstance: " + owningInstanceName);
        }
        List<FederatedQueryListElement> sessionIds = getSessionIds(owningInstanceName);
        sendReplicasList(sessionIds, owningInstanceName, MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_RESPONSE);
    }

    public void processRollingupgradegetidsforactivecachereconciliationresponse(ReplicationState replicationState) {
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradegetidsforactivecachereconciliationresponse received from " + instanceName);
        }
        ownedIdsForActiveCacheReconciliation.putIfAbsent(instanceName, replicationState);
        getIdsForActiveCacheReconciliationDoneSignal.countDown();
    }   
   
    // temporarily keep the migratedHttpSessions until they successully reach the destination.
    ConcurrentHashMap<String, Set<String>> migratedHttpSessionIds =
            new ConcurrentHashMap<String, Set<String>>();

    public void processReconcilehttpsessionrequest(ReplicationState reconcileRequest) {
        Set<String> migratedSessionIds = RollingUpgradeUtil.processActiveCacheReconciliationRequest(reconcileRequest,
                getSingletonStore(),
                RECONCILE_HTTPSESSION_RESPONSE, isReplicationCompressionEnabled());
        // keep the migrated HttpSessions and passivate them once they are activated in the remote instance.
        migratedHttpSessionIds.put(reconcileRequest.getInstanceName(), migratedSessionIds);
    }

    public void processReconcilehttpsessionresponse(ReplicationState reconcileResponse) {
        reconciledActiveHttpSessions.put(reconcileResponse.getInstanceName(),
                reconcileResponse);
        activeHttpSessionReconcileSignal.get().countDown();
    }

    public void processReconcilehttpsessionreplicarequest(ReplicationState reconcileRequest) {
        RollingUpgradeUtil.processReplicaCacheReconciliationRequest(reconcileRequest,
                sessions,
                RECONCILE_HTTPSESSION_REPLICA_RESPONSE, isReplicationCompressionEnabled());
    }

    public void processReconcilehttpsessionreplicaresponse(ReplicationState reconcileResponse) {
        reconciledReplicaHttpSessions.put(reconcileResponse.getInstanceName(),
                reconcileResponse);
        replicaHttpSessionReconcileSignal.get().countDown();
    }

    public void processPostreconcilehttpsessionrequest(ReplicationState reconcileRequest) {
        boolean saveMigratedSessionToReplicaCache = EEPersistenceTypeResolver.REPLICATED_TYPE.
                equalsIgnoreCase(getPassedInPersistenceType());
        RollingUpgradeUtil.processPostActiveCacheReconciliationRequest(reconcileRequest,
                getSingletonStore(),
                migratedHttpSessionIds.get(reconcileRequest.getInstanceName()),
                saveMigratedSessionToReplicaCache,
                POST_RECONCILE_HTTPSESSION_RESPONSE);
    }

    public void processPostreconcilehttpsessionresponse(ReplicationState reconcileResponse) {
        remotelyLockedHttpSessions.put(reconcileResponse.getInstanceName(), reconcileResponse);
        postHttpSessionReconcileSignal.get().countDown();
    }


    private void reconcileSessionActiveCache(long waitTime, RollingUpgradeContext ctx) {
        if (!replicationUtil.isInstanceLoadBalancedByCLB() || !isConverged()) {
            ctx.addMessage(getApplicationId(),
                    "Active cache reconciliation skipped for HttpSession. " +
                            "isConverged = " + isConverged() + ", isInstanceLoadBalancedByCLB = " +
                            replicationUtil.isInstanceLoadBalancedByCLB());
            return;
        }
        long startTime = System.currentTimeMillis();
        ReplicationStore storeImpl = getSingletonStore();

        int reconciledSessions = RollingUpgradeUtil.doActiveCacheReconciliation(
                storeImpl, RECONCILE_HTTPSESSION_REQUEST, activeHttpSessionReconcileSignal,
                waitTime, reconciledActiveHttpSessions);
        long timeTaken = (System.currentTimeMillis() - startTime) / 1000;

        int suspectSessions = RollingUpgradeUtil.doPostActiveCacheReconcilation(
                storeImpl, POST_RECONCILE_HTTPSESSION_REQUEST,
                postHttpSessionReconcileSignal, waitTime - timeTaken, remotelyLockedHttpSessions);

        timeTaken = System.currentTimeMillis() - startTime;

        ctx.addMessage(getApplicationId(), "Active cache reconciliation completed for HttpSession. " +
                " No. of sessions reconciled = " + reconciledSessions +
                ", No. of sessions still in suspect = " + suspectSessions +
                ", Size of active cache = " + sessions.size() +
                ", Size of expat = " + expatIdsMap.size() + ", Time taken (ms) = " + timeTaken);

    }

    protected void doTriggerReplicaCacheReconciliation(long waitTime, boolean isInstanceLoadBalancedByCLB) {
        if(!isInstanceLoadBalancedByCLB) {
            //trigger replicateFrom partner to do
            //replica cache reconciliation
            ReplicationHealthChecker healthChecker
                = ReplicationHealthChecker.getInstance();
            String replicateFromInstanceName = healthChecker.getReshapeReplicateFromInstanceName();
            sendRollingUpgradeAdvisory(waitTime, replicateFromInstanceName);
        } else {
            //trigger all partners to do
            //replica cache reconciliation
            sendRollingUpgradeAdvisory(waitTime);
        }
    }

    public void triggerReplicaCacheReconciliation(long waitTime, RollingUpgradeContext ctx) {
        int numberOfRespondants = 1;
        if(ReplicationUtil.checkIsInstanceLoadBalancedByCLB()) {
            numberOfRespondants = ReplicationUtil.getNumberExpectedRespondants();
        }
        long startTime = System.currentTimeMillis();
        if (_logger.isLoggable(Level.FINE)) {
            startTime = System.currentTimeMillis();
        }
        reconcileReplicaCacheDoneSignal = new CountDownLatch(numberOfRespondants);
        //trigger replicateFrom partner(s) to do
        //replica cache reconciliation
        doTriggerReplicaCacheReconciliation(waitTime, ReplicationUtil.checkIsInstanceLoadBalancedByCLB());
        try {
            reconcileReplicaCacheDoneSignal.await(waitTime, TimeUnit.SECONDS);
        } catch(InterruptedException ex) {
            ;
        } finally {
            if(reconcileReplicaCacheDoneSignal.getCount() != 0) {
                String errorMsg = "ReplicationManagerBase>>triggerReplicaCacheReconciliation timed out after " + waitTime + " seconds";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationManagerBase triggerReplicaCacheReconciliation successful after a wait: wait time = " + (System.currentTimeMillis() - startTime));
                }
            }
        }
    }

    protected ConcurrentHashMap<String, FederatedQueryListElement> processCacheReconciliationResults(ConcurrentHashMap<String, ReplicationState> ownedIdsForReplicaCacheReconciliation) {
        ConcurrentHashMap<String, FederatedQueryListElement> result =
            new ConcurrentHashMap<String, FederatedQueryListElement>();
        Collection<ReplicationState> ownedReplicasList = ownedIdsForReplicaCacheReconciliation.values();
        Iterator<ReplicationState> it = ownedReplicasList.iterator();
        while(it.hasNext()) {
            ReplicationState nextOwnedListReplicationState = it.next();
            List<FederatedQueryListElement>nextOwnedListOfStates
                = (List<FederatedQueryListElement>)RollingUpgradeUtil.getObject(nextOwnedListReplicationState.getState());
            for(int i=0; i<nextOwnedListOfStates.size(); i++) {
                FederatedQueryListElement nextElement = nextOwnedListOfStates.get(i);
                String nextElementId = nextElement.getId();
                if(nextElementId != null) {
                    FederatedQueryListElement existingElement = result.get(nextElementId);
                    if(existingElement == null || existingElement.getVersion() < nextElement.getVersion()) {
                        result.put(nextElementId, nextElement);
                    }
                }
            }
        }
        return result;
    }    

    /**
     * reconcile the replica cache (of your replica partner)
     * query instance1 to get a list of replica id/version data elements
     * then do 2 iterations:
     * iterate over the query result:
     * if an id from this list does not exist in this active cache
     * issue a remove message & load acknowledgment
     * if an id exists and the versions match do nothing
     * if an id exists and the active version is > replica version,
     *   - do a save
     * iterate over the active cache
     * if an id from active cache does not exist in the replica list
     *   - do a save
     * @param waitTime the waitTime
     */
    public void doReplicaCacheReconciliation(long waitTime, RollingUpgradeContext ctx) {
        reconcileSessionReplicaCache(waitTime, ctx);
    }

    public ConcurrentHashMap<String, FederatedQueryListElement> triggerGetIdsForReplicaCacheReconciliation(long waitTime) {
        long startTime = 0L;
        if (_logger.isLoggable(Level.FINE)) {
            startTime = System.currentTimeMillis();
        }
        //sending to all for both types of load balancer
        int numberOfRespondants = ReplicationUtil.getNumberExpectedRespondants();
        getIdsForReplicaCacheReconciliationDoneSignal = new CountDownLatch(numberOfRespondants);
        //trigger replicateTo partner(s) to
        //provide owned session ids list(s)
        ownedIdsForReplicaCacheReconciliation
            = new ConcurrentHashMap<String, ReplicationState>();
        doTriggerGetIdsForReplicaCacheReconciliation(ReplicationUtil.checkIsInstanceLoadBalancedByCLB());
        ConcurrentHashMap<String, FederatedQueryListElement> result = null;
        try {
            getIdsForReplicaCacheReconciliationDoneSignal.await(waitTime, TimeUnit.SECONDS);
            result = processCacheReconciliationResults(ownedIdsForReplicaCacheReconciliation);
        } catch(InterruptedException ex) {
            result = new ConcurrentHashMap<String, FederatedQueryListElement>();
        } finally {
            if(getIdsForReplicaCacheReconciliationDoneSignal.getCount() != 0) {
                _logger.log(Level.WARNING, "ReplicationManagerBase>>triggerGetIdsForReplicaCacheReconciliation timed out after "
                    + waitTime + " seconds");
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationManagerBase triggerGetIdsForReplicaCacheReconciliation successful after a wait: wait time = " + (System.currentTimeMillis() - startTime));
                }
            }
            ownedIdsForReplicaCacheReconciliation = null;
        }
        return result;
    }

    protected void doTriggerGetIdsForReplicaCacheReconciliation(boolean isInstanceLoadBalancedByCLB) {
        //trigger all partners to do
        //replica cache reconciliation
        sendRollingUpgradeGetIdsForReplicaCacheReconciliationAdvisory();
    }    
   
    void cleanOutZombieReplicas() {
        //todo for now issue load acknowledgement for each
        //active cache entry
        //this is functionally correct but not optimal
        //solution will be to improve query for replica
        //ids to track the source instance and then only
        //issue load acks to those ids that are not from
        //our replicate to partner
        BaseHASession sess = null;
        //iterate over active sessions cache
        //take copy asap to avoid synchronizing too long
        List sessionsCopiedList = getClonedSessionsList();
        Iterator it = sessionsCopiedList.iterator();
        while(it.hasNext()) {
            sess = (BaseHASession)it.next();
            this.sendLoadAcknowledgement(sess.getIdInternal(), sess.getVersion(), sess.getBeKey());
        }
    }

    public void processRollingupgradegetidsforreplicacachereconciliationadvisory(ReplicationState replicationState) {
        String instanceName = ReplicationUtil.getInstanceName();
        String owningInstanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processRollingupgradegetidsforreplicacachereconciliationadvisory:instance: " + instanceName);
            _logger.fine("in " + this.getClass().getName() + ">>processRollingupgradegetidsforreplicacachereconciliationadvisory:owningInstance: " + owningInstanceName);
        }
        List<FederatedQueryListElement> sessionIds = getSessionIds(owningInstanceName);
        sendReplicasList(sessionIds, owningInstanceName, MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_RESPONSE);
    }

    public void processRollingupgradegetidsforreplicacachereconciliationresponse(ReplicationState replicationState) {
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradegetidsforreplicacachereconciliationresponse received from " + instanceName);
        }
        ownedIdsForReplicaCacheReconciliation.putIfAbsent(instanceName, replicationState);
        getIdsForReplicaCacheReconciliationDoneSignal.countDown();
    }   
   
    private void reconcileSessionReplicaCache(long waitTime, RollingUpgradeContext ctx) {
        long startTime = System.currentTimeMillis();
        ReplicationStore storeImpl = getSingletonStore();
        int reconciledSessions = RollingUpgradeUtil.doReplicaCacheReconciliation(
                storeImpl, RECONCILE_HTTPSESSION_REPLICA_REQUEST,
                replicaHttpSessionReconcileSignal, waitTime, reconciledReplicaHttpSessions);
        long timeTaken = System.currentTimeMillis() - startTime;
        ctx.addMessage(getApplicationId(), "Replica cache reconciliation completed for HttpSession. " +
                " No of sessions reconciled = " + reconciledSessions +
                ", Size of replica cache = " + getReplicatedSessions().getEntryCount() +
                ", Time taken (ms) = " + timeTaken);
    }
   
    protected void sendLoadAcknowledgement(String id, long version, String beKey) {
        JxtaReplicationSender sender
            = JxtaReplicationSender.createInstance();
        ReplicationState loadReceivedState =
            ReplicationState.createBroadcastLoadReceivedState(MODE_WEB, id, this.getApplicationId(), version, ReplicationUtil.getInstanceName(), MESSAGE_BROADCAST_LOAD_RECEIVED);
        ReplicationUtil repUtil = ReplicationUtil.createReplicationUtil();
        if ((beKey != null) && repUtil.isInstanceLoadBalancedByCLB()){
            loadReceivedState.setProperty(ReplicationState.IGNORE_REMOVE_INSTANCE_NAME,
                repUtil.getFailoverServerInstanceForBeKey(beKey));
        } else {
            ReplicationHealthChecker healthChecker = ReplicationHealthChecker.getInstance();
            String currentReplicaPartner = healthChecker.getCurrentPartnerInstanceName();
                loadReceivedState.setProperty(ReplicationState.IGNORE_REMOVE_INSTANCE_NAME,
                currentReplicaPartner);
        }
        sender.sendBroadcastQuery(loadReceivedState);
    }
   
    protected void readSessions(ObjectInputStream ois)
            throws ClassNotFoundException, IOException {
        int count = ois.readInt();
        synchronized (sessions) {
            for(int i=0; i<count; i++) {
                BaseHASession nextSession = (BaseHASession)ois.readObject();
                sessions.put(nextSession.getIdInternal(), nextSession);
            }
        }
    }    
   
    protected void readReplicatedSessionUpdates(ObjectInputStream ois)
        throws ClassNotFoundException, IOException {
        replicaCache.readReplicatedSessionUpdates(ois);
    }
   
    protected void writeSessions(ObjectOutputStream oos)
            throws IOException {
        synchronized (sessions) {
            ReplicationUtil.writeHashMap(sessions, oos);
        }
    }   
   
    protected void writeReplicatedSessionUpdates(ObjectOutputStream oos)
        throws IOException {
        replicaCache.writeReplicatedSessionUpdates(oos);
    }

    List<FederatedQueryListElement> getSessionIds(String owningInstanceName) {
        List<FederatedQueryListElement> sessionIds = new ArrayList<FederatedQueryListElement>();
        //using set to avoid dups
        HashSet sessionIdsSet = new HashSet();
        List replicasToRemove = new ArrayList();
        ReplicationHealthChecker healthChecker = ReplicationHealthChecker.getInstance();
        String correctBuddyReplicaName
            = healthChecker.getReshapeReplicateToInstanceName(null, owningInstanceName, 0L);
        //iterate over http replicas
        Iterator it = getReplicatedSessions().values();
        while(it.hasNext()) {
            ReplicationState nextState
                = (ReplicationState)it.next();
            if(!RollingUpgradeUtil.filterOwnership(owningInstanceName, correctBuddyReplicaName, nextState, sessionIdsSet)) {
                //we remove this entry if it is not on the correct replica partner - i.e. it is a zombie
                replicasToRemove.add(nextState.getId());
            }
        }
        //remove the zombies here
        for(int i=0; i<replicasToRemove.size(); i++) {
            String nextId = (String)replicasToRemove.get(i);
            if(nextId != null) {
                removeFromReplicationCache(nextId);
            }
        }
        sessionIds.addAll(sessionIdsSet);
        return sessionIds;
    }

    List<FederatedQueryListElement> getSessionIdsThirdPartySPI(String owningInstanceName) {
        BackingStore backingStore
            = this.getBackingStore();
        HttpSessionExtraParams httpExtraParamCriteria
            = HttpSessionExtraParams.createSearchCriteria(this, owningInstanceName);
        Collection<HttpSessionExtraParams> httpColl
            = backingStore.findByCriteria(null, httpExtraParamCriteria);

        List<FederatedQueryListElement> sessionIds = new ArrayList<FederatedQueryListElement>();
        //using set to avoid dups
        HashSet sessionIdsSet = new HashSet();
        Iterator<HttpSessionExtraParams> httpResults =
            (Iterator<HttpSessionExtraParams>) httpColl.iterator();
        while (httpResults.hasNext()) {
            HttpSessionExtraParams eParam = httpResults.next();
            if (owningInstanceName.equals(eParam.getCurrentOwnerInstanceName())) {
                sessionIdsSet.add(new FederatedQueryListElement((String)eParam.getId(), -1L, ReplicationUtil.getInstanceName()));
            }
        }
        sessionIds.addAll(sessionIdsSet);
        return sessionIds;
    }
   
    /**
     * send a rolling upgrade advisory message to instance to trigger
     * it to do rolling upgrade reconciliation for the sending
     * instance
     *
     * @param waitTime the waitTime
     * @param instanceName the instance to be sent the rolling upgrade advisory
     */
    public void sendRollingUpgradeAdvisory(long waitTime, String instanceName) {

        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationManagerBase",
                             "sendRollingUpgradeAdvisory",
                             new Object[] {instanceName});
        }
        try {
            RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_WEB, MESSAGE_BROADCAST_ROLLING_UPGRADE_ADVISORY, instanceName, waitTime);
        } catch (IOException ex) {}
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationManagerBase",
                            "sendRollingUpgradeAdvisory");
        }
    }

    /**
     * send a rolling upgrade advisory message to instance to trigger
     * it to do rolling upgrade active cache restoration for the sending
     * instance
     *
     * @param instanceName the instance to be sent the rolling upgrade advisory
     */
    public void sendRollingUpgradeActiveCacheRestorationAdvisory(String instanceName) {

        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationManagerBase",
                             "sendRollingUpgradeActiveCacheRestorationAdvisory",
                             new Object[] {instanceName});
        }
        try {
            RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_WEB, MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_ADVISORY, instanceName);
        } catch (IOException ex) {}
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationManagerBase",
                            "sendRollingUpgradeActiveCacheRestorationAdvisory");
        }
    }

    /**
     * broadcast a rolling upgrade active cache restoration advisory message to trigger
     * rolling upgrade active cache restoration for the sending
     * instance
     */
    protected void sendRollingUpgradeActiveCacheRestorationAdvisory() {
        //broadcast rolling upgrade active cache restoration advisory
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_WEB, MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_ADVISORY);
    }

    /**
     * send a rolling upgrade advisory message to instance to trigger
     * it to do rolling upgrade replica cache restoration for the sending
     * instance
     *
     * @param instanceName the instance to be sent the rolling upgrade advisory
     */
    public void sendRollingUpgradeReplicaCacheRestorationAdvisory(String instanceName) {
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this,
                MODE_WEB, MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_ADVISORY,
                "ReplicationManagerBase", "sendRollingUpgradeReplicaCacheRestorationAdvisory", instanceName);
        /*
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationManagerBase",
                             "sendRollingUpgradeReplicaCacheRestorationAdvisory",
                             new Object[] {instanceName});
        }
        try {
            RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_WEB, MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_ADVISORY, instanceName);
        } catch (IOException ex) {}
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationManagerBase",
                            "sendRollingUpgradeReplicaCacheRestorationAdvisory");
        }
        */
    }

    /**
     * broadcast a rolling upgrade active cache restoration advisory message to trigger
     * rolling upgrade replica cache restoration for the sending
     * instance
     */
    protected void sendRollingUpgradeReplicaCacheRestorationAdvisory() {
        //broadcast rolling upgrade active cache restoration advisory
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_WEB, MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_ADVISORY);
    }

    /**
     * broadcast a rolling upgrade advisory message to trigger
     * rolling upgrade reconciliation for the sending
     * instance
     * @param waitTime the waitTime
     */
    protected void sendRollingUpgradeAdvisory(long waitTime) {
        //broadcast rolling upgrade advisory
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_WEB, MESSAGE_BROADCAST_ROLLING_UPGRADE_ADVISORY, waitTime);
    }

    /**
     * send a rolling upgrade reconciliation complete advisory message to instance to signal
     * to the original caller instance that replica reconciliation is complete
     *
     * @param instanceName the instance to be sent the rolling upgrade reconciliation complete advisory
     */
    public void sendRollingUpgradeReplicaReconciliationCompleteAdvisory(String instanceName) {

        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationManagerBase",
                             "sendRollingUpgradeReplicaReconciliationCompleteAdvisory",
                             new Object[] {instanceName});
        }
        try {
            RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_WEB, MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_RECONCILIATION_COMPLETE_ADVISORY, instanceName);
        } catch (IOException ex) {}
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationManagerBase",
                            "sendRollingUpgradeReplicaReconciliationCompleteAdvisory");
        }
    }

    /**
     * broadcast a rolling upgrade getIds
     * for replica cache reconciliation advisory message
     *
     */
    protected void sendRollingUpgradeGetIdsForReplicaCacheReconciliationAdvisory() {
        //broadcast rolling upgrade getIds for replica cache reconciliation advisory
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_WEB, MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_ADVISORY);
    }   

    public void processRollingupgradeadvisory(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradeadvisory received from " + instanceName);
        }
        long waitTime = ((Long)replicationState.getProperty(ReplicationState.WAIT_TIME)).longValue();
        //we have been triggered to do replica
        //cache reconciliation for our replicateTo partner
        doReplicaCacheReconciliation(waitTime, null); // TODO :: check last param, this whole method is unused -- remove it later.
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradeadvisory sending reconciliation completed advisory to: " + instanceName);
        }
        sendRollingUpgradeReplicaReconciliationCompleteAdvisory(instanceName);
    }

    public void processRollingupgradeactivecacherestorationadvisory(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradeactivecacherestorationadvisory received from " + instanceName);
        }
        List<ReplicationState> ownedReplicas = getSessionReplicas(instanceName);
        sendReplicationStateList(ownedReplicas, instanceName, MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_RESPONSE);

    }

    public void processRollingupgradeactivecacherestorationresponse(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradeactivecacherestorationresponse received from " + instanceName);
        }
        ownedReplicaListsReceivedForActiveCacheRestoration.putIfAbsent(replicationState.getInstanceName(), replicationState);
        restoreActiveCacheDoneSignal.countDown();
    }

    public void processRollingupgradereplicacacherestorationadvisory(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradereplicacacherestorationadvisory received from " + instanceName);
        }
        List<ReplicationState> ownedReplicas = getActiveSessionsToReplicateTo(instanceName);
        sendReplicationStateList(ownedReplicas, instanceName, MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_RESPONSE);

    }

    public void processRollingupgradereplicacacherestorationresponse(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradereplicacacherestorationresponse received from " + instanceName);
        }
        ownedReplicaListsReceivedForReplicaCacheRestoration.putIfAbsent(replicationState.getInstanceName(), replicationState);
        restoreReplicaCacheDoneSignal.countDown();
    }

    public void processRollingupgradereplicareconciliationcompleteadvisory(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradereplicareconciliationcompleteadvisory received from " + replicationState.getInstanceName());
        }       
        reconcileReplicaCacheDoneSignal.countDown();
    }

    List<ReplicationState> getSessionReplicas(String owningInstanceName) {
        List<ReplicationState> sessionIds = new ArrayList();
        //using set to avoid dups
        HashSet sessionIdsSet = new HashSet();
        //iterate over http replicas
        Iterator it = getReplicatedSessions().values();
        while(it.hasNext()) {
            ReplicationState nextState
                = (ReplicationState)it.next();
            RollingUpgradeUtil.filterOwnershipOfReplicas(owningInstanceName, nextState, sessionIdsSet);
        }
        sessionIds.addAll(sessionIdsSet);
        return sessionIds;
    }

    List<ReplicationState> getActiveSessionsToReplicateTo(String owningInstanceName) {
        ReplicationStore store = this.getSingletonStore();
        List<ReplicationState> sessionIds = new ArrayList();
        //using set to avoid dups
        HashSet sessionIdsSet = new HashSet();
        //iterate over active sessions cache
        //take copy asap to avoid synchronizing too long
        List sessionsCopiedList = getClonedSessionsList();
        Iterator it = sessionsCopiedList.iterator();
        while(it.hasNext()) {
            BaseHASession nextSession = (BaseHASession)it.next();
            String beKey = nextSession.getBeKey();
            if(RollingUpgradeUtil.shouldReplicateTo(beKey, owningInstanceName)) {
                ReplicationState nextState = null;
                try {
                    nextState = store.getTransmitState(nextSession);
                } catch (IOException ex) {}
                if(nextState != null) {
                    sessionIdsSet.add(nextState);
                }
            }

        }
        sessionIds.addAll(sessionIdsSet);
        return sessionIds;
    }

    private void sendReplicationStateList(List<ReplicationState> ownedReplicas,
                               String instanceName, String command) {
        byte[] ownedReplicasState = null;
        try {
            ownedReplicasState = ReplicationUtil.getByteArray((Serializable)ownedReplicas);
        } catch (IOException ex) {}
        //need a unique pseudo-id for this query
        String appId = getApplicationId();
        String id = appId + unique.getAndIncrement();
        ReplicationState ownedReplicasListState = new ReplicationState(MODE_WEB,
                id, appId, 0L, 0L, 0L, null, null, ReplicationUtil.getInstanceName(),
                command, ownedReplicasState, null, null);
        if (ownedReplicas == null || ownedReplicas.isEmpty()) {
            ownedReplicasListState.setNack(true);
        }
        JxtaSocketChannel channel = JxtaSocketChannel.getInstance();
        channel.send(ownedReplicasListState, instanceName);
    }

    private void sendReplicasList(List<FederatedQueryListElement> ownedReplicas,
                               String instanceName, String command) {
        byte[] ownedReplicasState = null;
        try {
            ownedReplicasState = ReplicationUtil.getByteArray((Serializable)ownedReplicas);
        } catch (IOException ex) {}
        //need a unique pseudo-id for this query
        String appId = getApplicationId();
        String id = appId + unique.getAndIncrement();
        ReplicationState ownedReplicasListState = new ReplicationState(MODE_WEB,
                id, appId, 0L, 0L, 0L, null, null, ReplicationUtil.getInstanceName(),
                command, ownedReplicasState, null, null);
        if (ownedReplicas == null || ownedReplicas.isEmpty()) {
            ownedReplicasListState.setNack(true);
        }
        JxtaSocketChannel channel = JxtaSocketChannel.getInstance();
        channel.send(ownedReplicasListState, instanceName);
    }    
   
    // end rolling upgrade related code   
   
    /**
     * Clear all sessions from the manager
     */
    public void clearSessions() {
        //un-register this app from ReplicationMessageRouter
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicationManagerBase>>clearSessions:getApplicationId()" + getApplicationId());
        }
        ReplicationMessageRouter router = getRouter();
        if(router != null) {
            router.removeReplicationManager(getApplicationId());
        }
    }
   
    /**
     * Clear all sessions from the Store.
     * in this case a no-op
     */
    public void clearStore() {
        //deliberate no-op
        return;
    }
    
    protected ReplicationSessionMonitors replicatedSessionMonitors = null;

    /**
    * get the replicated sessions cache
    */
    public BaseCache getReplicatedSessions() {
        return replicaCache.getReplicatedSessions();
    }
   
    void printReplicatedSessionIds() {
        replicaCache.printReplicatedSessionIds();
    }

    /**
     * Put session State in replica cache
     * @param sessionState
     */   
    protected void putInReplicationCache(ReplicationState sessionState) {
        replicaCache.putInReplicationCache(sessionState);
    }
   
    public Object getReplicationSessionMonitor(String id) {
        return replicatedSessionMonitors.get(id);
    }   
  
    /**
     * get session State from replica cache based on the id
     * @param id
     */   
    protected ReplicationState getFromReplicationCache(String id) {
        return replicaCache.getFromReplicationCache(id);
    }

    /**
     * remove session State from replica cache based on the id of sessionState
     * @param sessionState
     */   
    protected void removeFromReplicationCache(ReplicationState sessionState) {
        if(sessionState == null) {
            return;
        }
        removeFromReplicationCache((String)sessionState.getId());
    }
  
    /**
     * remove session State from replica cache based on the id
     * @param id
     */   
    protected ReplicationState removeFromReplicationCache(String id) {
        return replicaCache.removeFromReplicationCache(id);
    }

    /**
     * remove session State from replica cache based on the id and return it
     * @param id
     */    
    protected ReplicationState transferFromReplicationCache(String id) {
        return replicaCache.transferFromReplicationCache(id);
    }

    public void activate(Session session, boolean removeFromExpat) {
        this.add(session);

        if(removeFromExpat) {
            removeExpatListElement(session.getId());
        }
    }

    public Map getActiveHttpSessions() {
        return sessions;
    }

    /**
     * Add this Session to the set of active Sessions for this Manager.
     * This method checks for an existing version and avoids overriding
     * a later version with an earlier version
     *
     * @param session Session to be added
     */
    @Override
    public void add(Session session) {
        BaseHASession addedSession = (BaseHASession)session;
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>add: " + session.getIdInternal() + " version: " + addedSession.getVersion());
        }
        String id = (String)session.getIdInternal();
        BaseHASession currentSession = null;
        Object monitor = getReplicationSessionMonitor(id);
        synchronized (monitor) {
            synchronized (sessions) {
                currentSession = (BaseHASession) sessions.get(id);
                if ((currentSession != null) && currentSession.getVersion() >= addedSession.getVersion()) {
                    return;
                }
                super.add(session);
            }
        }
        if (_salogger.isLoggable(Level.FINE)) {
            String replacedVersion = "";
            if (currentSession != null) {
                replacedVersion = ". Replaced version " + currentSession.getVersion();
            }
            _salogger.fine("Put in active cache appId=" + this.getApplicationId() + " id:" + session.getIdInternal() +
                    "[ver:" + addedSession.getVersion() + "] of instance " + ReplicationUtil.getInstanceName() +
                    replacedVersion);
        }
    }

    /**
     * Remove this Session from the active Sessions for this Manager.
     *
     * @param session Session to be removed
     */
    @Override
    public void remove(Session session) {
        Object monitor = getReplicationSessionMonitor(
            session.getIdInternal());
        synchronized (monitor) {
            // super.remove already synchronizes on "sessions"
            super.remove(session);
        }
    }    
   
    /**
    * Our Replicator instance (for SimpleMetadata)
    */
    protected BackingStore backingStore = null;
   
    /**
    * Our Replicator instance (for CompositeMetadata)
    */
    protected BackingStore compositeBackingStore = null;   
   
    /**
    * get the backingStore
    */
    public BackingStore getBackingStore() {
        if(backingStore == null) {
            this.createBackingStore();
        }
        return backingStore;
    }
   
    /**
    * get the backingStore (for CompositeMetadata)
    */
    public BackingStore getCompositeBackingStore() {
        if(compositeBackingStore == null) {
            this.createCompositeBackingStore();
        }
        return compositeBackingStore;
    }   
   
    /**
    * set the backing store
    * @param aBackingStore
    */
    public void setBackingStore(BackingStore aBackingStore) {
        backingStore = aBackingStore;
    }
   
    /**
    * set the backing store for CompositeMetadata
    * @param aBackingStore
    */
    public void setCompositeBackingStore(BackingStore aBackingStore) {
        compositeBackingStore = aBackingStore;
    }
   
    boolean isThirdPartyBackingStoreInUse() {
        BackingStore backingStore
            = getBackingStore();
        return (!(backingStore instanceof JxtaBackingStoreImpl));
    }   

    /**
    * create and set the backing store
    */    
    void createBackingStore() {
        BackingStoreFactory storeFactory = new JxtaBackingStoreFactory();
        BackingStoreRegistry backingStoreRegistry
            = BackingStoreRegistry.getInstance();
        Properties inputEnv
            = backingStoreRegistry.getFactoryClassEnv(getPassedInPersistenceType());
        Properties env = (Properties)inputEnv.clone();
        //does this manager & backing store support duplicate id semantics
        //for batch replication usage
        env.put(DUPLICATE_IDS_SEMANTICS_PROPERTY, Boolean.valueOf(this.isDuplicateIdsSemanticsAllowed()));
        env.put(SUPPRESS_LOAD_ACK_PROPERTY, Boolean.TRUE);
        env.put(SESSION_MANAGER_PROPERTY, this); // this class itself does not implement ReplicationManager, but all concrete subclasses do implement ReplicationManager, so it should be ok.
        BackingStore backingStore = null;
        try {
            backingStore = storeFactory.createBackingStore(
                        this.getApplicationId(), //appid
                        String.class,
                        SimpleMetadata.class,     //type
                        env);
        } catch (BackingStoreException ex) {
            //deliberate no-op
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("backingStore: " + backingStore);
        }        
        if(backingStore != null) {
            if(backingStore instanceof JxtaBackingStoreImpl) {
                ((JxtaBackingStoreImpl)backingStore).setMode(MODE_WEB);
            }       
            this.setBackingStore(backingStore);
        }
    }

    /**
    * create and set the backing store for CompositeMetadata
    */    
    void createCompositeBackingStore() {
        BackingStoreFactory storeFactory = new JxtaBackingStoreFactory();
        BackingStoreRegistry backingStoreRegistry
            = BackingStoreRegistry.getInstance();
        Properties env
            = backingStoreRegistry.getFactoryClassEnv(getPassedInPersistenceType());
        env.put(DUPLICATE_IDS_SEMANTICS_PROPERTY, Boolean.valueOf(this.isDuplicateIdsSemanticsAllowed()));
        env.put(SESSION_MANAGER_PROPERTY, this); // this class itself does not implement ReplicationManager, but all concrete subclasses do implement ReplicationManager, so it should be ok.       
        BackingStore compositeBackingStore = null;
        try {
            compositeBackingStore = storeFactory.createBackingStore(
                        this.getApplicationId(), //appid
                        String.class,
                        CompositeMetadata.class,     //type
                        env);
        } catch (BackingStoreException ex) {
            //deliberate no-op
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("compositeBackingStore: " + compositeBackingStore);
        }        
        if(compositeBackingStore != null) {
            if(compositeBackingStore instanceof JxtaBackingStoreImpl) {
                ((JxtaBackingStoreImpl)compositeBackingStore).setMode(MODE_WEB);
            }       
            this.setCompositeBackingStore(compositeBackingStore);
        }
    }    

    /**
    * return the backing store factory
    */    
    protected BackingStoreFactory getBackingStoreFactory() {
        BackingStoreRegistry backingStoreRegistry
            = BackingStoreRegistry.getInstance();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("getBackingStoreFactory:passedInPersistenceType=" + getPassedInPersistenceType());
        }        
        String factoryClassName
            = backingStoreRegistry.getFactoryClassName(this.getPassedInPersistenceType());
        return getBackingStoreFactoryFromName(factoryClassName);
    }

    /**
    * helper method to return the backing store factory based on className
    * @param className
    */   
    private BackingStoreFactory getBackingStoreFactoryFromName(String className) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("getBackingStoreFactoryFromName:className: " + className);
        }       
        BackingStoreFactory backingStoreFactory = null;
        try {
            backingStoreFactory =
                (BackingStoreFactory) (Class.forName(className)).newInstance();
        } catch (Exception ex) {
            //FIXME - throw exception?
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("unable to create backing store factory");
            }           
        }
        return backingStoreFactory;
    }   
   
    /** create and return a new session; delegates to session factory */
    protected Session createNewSession() {
        Session sess = getSessionFactory().createSession(this);
        return sess;
    }
   
    /**
     * Get new session class (new method in ManagerBase that we over-ride
     * instead of createNewSession()
     */
    public Session createEmptySession() {      
        Session sess = getSessionFactory().createSession(this);
        return sess;
    }
   
    /**
    * get the application id
    */        
    public String getApplicationId() {
        HAStoreBase store = (HAStoreBase)getStore();
        return store.getApplicationId();
    }
   
    /** subclasses will over-ride this method */
    public String getMonitorAttributeValues() {
        return "testing..1..2..3";
    }
   
    public int getSessionsCacheSize() {
        return sessions.size();
    }   
   
    /**
     * Return the active Session, associated with this Manager, with the
     * specified session id (if any); otherwise return <code>null</code>.
     * This method checks the persistence store if persistence is enabled,
     * otherwise just uses the functionality from ManagerBase.
     *
     * @param id The session id for the session to be returned
     *
     * @exception IllegalStateException if a new session cannot be
     *  instantiated for any reason
     * @exception IOException if an input/output error occurs while
     *  processing this request
     */
    public Session findSession(String id) throws IOException {
  Object monitor = getReplicationSessionMonitor(id);
  synchronized (monitor) {
            if(!this.isSessionIdValid(id)) {
                return null;
            }       
            WebModuleStatistics stats = this.getWebModuleStatistics();
            Session session = findSessionFromCacheOnly(id);
            if (session != null) {
                if(isMonitoringEnabled()) {
                    stats.processCacheHit(true);
                }
                return (session);
            }
   
            // See if the Session is in the Store
            if(isMonitoringEnabled()) {
                stats.processCacheHit(false);
            }
            LoadProcessingGovernor.incrementCurrentLoads();
            session = swapIn(id);
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("IN ReplicationManagerBase>>findSession: returned sess = " + (BaseHASession)session);       
            }
            return (session);
        }
    }
   
    /**
     * This method is deliberately over-riding this same method
     * in PersistentManagerBase.  It is ignoring the removeCachedCopy
     * parameter.  (see org.apache.catalina.session.PersistentManagerBase
     * for more details on this method).
     *
     * @param id The session id for the session to be returned
     * @param removeCachedCopy
     *
     * @exception IllegalStateException if a new session cannot be
     *  instantiated for any reason
     * @exception IOException if an input/output error occurs while
     *  processing this request
     */
    public Session findSession(String id, boolean removeCachedCopy) throws IOException {
        if(!this.isSessionIdValid(id)) {
            return null;
        }       
        Session theSession = this.findSession(id);  
        return theSession;
    }
   
    /**
     * This method is a clone of the ManagerBase>>findSession.
     * It is used here to avoid problems with calling super.findSession
     * from this class.
     * Return the active Session, associated with this Manager, with the
     * specified session id (if any); otherwise return <code>null</code>.
     *
     * @param id The session id for the session to be returned
     *
     * @exception IllegalStateException if a new session cannot be
     *  instantiated for any reason
     * @exception IOException if an input/output error occurs while
     *  processing this request
     */ 
    public Session findSessionFromCacheOnly(String id) throws IOException {

        if (id == null)
            return (null);
        if(this.isActiveCacheReconciliationOngoing()) {
            return findSessionFromCacheOnlyDuringReconciliation(id);
        } else {
            return findSessionFromCacheOnlyNonReconciliation(id);
        }

    }   
   
    /**
     * This method is a clone of the ManagerBase>>findSession.
     * It is used here to avoid problems with calling super.findSession
     * from this class.
     * Return the active Session, associated with this Manager, with the
     * specified session id (if any); otherwise return <code>null</code>.
     *
     * @param id The session id for the session to be returned
     *
     * @exception IllegalStateException if a new session cannot be
     *  instantiated for any reason
     * @exception IOException if an input/output error occurs while
     *  processing this request
     */
    public Session findSessionFromCacheOnlyNonReconciliation(String id) throws IOException {

        if (id == null) {
            return (null);
        }
        synchronized (sessions) {
            return sessions.get(id);
        }
    }
   
    public Session findSessionFromCacheOnlyDuringReconciliation(String id) throws IOException {
        Object monitor = getReplicationSessionMonitor(id);
        synchronized (monitor) {
            BaseHASession session = null;
            synchronized (sessions) {
                session = (BaseHASession) sessions.get(id);
            }
            if(session == null || !session.isSuspect()) {
                //if null just return it
                //if not suspect it has already been checked
                return session;
            }
            if(RollingUpgradeUtil.canUseSuspectSession(session,
                    expatIdsMap, expatReadWriteLock)) {
                //if it passes the check return it
                return session;
            } else {
                removeSessionFromManagerCache(session);
                return null;
            }
        }
    }
   
    public void repair(long repairStartTime) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicationManager>>repair");                      
        }        
        if(ReplicationHealthChecker.isStopping()) {
            return;
        }

        Session sessions[] = findSessions();

        for (int i = 0; i < sessions.length; i++) {
            StandardSession session = (StandardSession) sessions[i];
            if(session.getIsValid()
                    && (session.getIdInternal() != null)
                    && !session.hasExpired()
                    && isSessionOlderThan(session, repairStartTime)) {
                if(session.lockBackground()) {
                    try {
                        ((HASession)session).setPersistent(false);
                        ((HASession)session).setDirty(true, false);
                        doValveSave(session);
                    } finally {
                        session.unlockBackground();
                    }
                }                               
      }           
        }
    }       
   
    public void repair(long repairStartTime, boolean checkForStopping) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicationManagerBase>>repair: checkForStopping: " + checkForStopping);
        }        
        if(checkForStopping && ReplicationHealthChecker.isStopping()) {
            return;
        }

        Session sessions[] = findSessions();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicationManagerBase>>repair sessions size = " + sessions.length);
        }        
        for (int i = 0; i < sessions.length; i++) {
            StandardSession session = (StandardSession) sessions[i];
            boolean condition = true;
            if(checkForStopping) {
                condition = (session.getIsValid()
                    && (session.getIdInternal() != null)
                    && !session.hasExpired()
                    && isSessionOlderThan(session, repairStartTime));
            } else {
                //during flush ignore age check
                condition = (session.getIsValid()
                    && (session.getIdInternal() != null)
                    && !session.hasExpired() );               
            }
            if(condition) {         
                if(session.lockBackground()) {
                    try {
                        ((HASession)session).setPersistent(false);
                        ((HASession)session).setDirty(true, false);
                        doValveSave(session);
                    } catch(Throwable t) {
                        // FIXME evaluate log level
                        if (_logger.isLoggable(Level.FINE)) {
                            _logger.log(Level.FINE, "Throwable occurred during force flush", t);
                        }
                        break;
                    } finally {
                        session.unlockBackground();
                    }
                }                               
      }           
        }
    }
   
    public void respondToFailure(String instanceName, boolean checkForStopping) {
        ; //no op
    }
   
    /**
    * called from valve; does the save of session
    *
    * @param session
    *   The session to store
    */   
    public void doValveSave(Session session) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationManagerBase",
                             "doValveSave");
        }       
        long startTime = 0L;
        if(isMonitoringEnabled()) {
            startTime = System.currentTimeMillis();
        }       
        ReplicationStore store = null;
        try {
            store = (ReplicationStore)getSessionStore();
            if(store != null) {
                store.valveSave(session);
            }
        } catch (IOException ex) {
            if (_logger.isLoggable(Level.FINE)) {
                _logger.log(Level.FINE, "exception occurred in doValveSave sessionId=" +
                            session.getIdInternal(), ex);
            }           
        } finally {
            this.putSessionStore(store);
            if(isMonitoringEnabled()) {
                long endTime = System.currentTimeMillis();
                long elapsedTime = (endTime - startTime);
                if(_logger.isLoggable(Level.FINEST)) {
                    _logger.finest("VALVE_TIME MILLIS = " + elapsedTime);
                }
                WebModuleStatistics stats = this.getWebModuleStatistics();
                stats.processValveSave(elapsedTime);
                if(_logger.isLoggable(Level.FINEST)) {
                    _logger.finest("VALVE_TIME MILLIS = " + (endTime - startTime));
                }
            }           
        }                
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationManagerBase",
                            "doValveSave");
        }
    }

    /**
     * Look for a session in the Store and, if found, restore
     * it in the Manager's list of active sessions if appropriate.
     * The session will be removed from the Store after swapping
     * in, but will not be added to the active session list if it
     * is invalid or past its expiration.
     */
    @Override
    protected Session doSwapIn(String id, String version) throws IOException {
        Object monitor = getReplicationSessionMonitor(id);
        synchronized (monitor) {
            return super.doSwapIn(id, version);
        }
    }

    ReplicationState doUnicastLoadTest(BaseHASession session) {
        //start unicast load test
        ReplicationState resultState = null;
        ReplicationHealthChecker healthChecker
            = ReplicationHealthChecker.getInstance();
        String partnerInstanceName
            = healthChecker.getReshapeReplicateToInstanceName(null, 3000L);
        ReplicationStore store = this.getSingletonStore();
        try {
            resultState = store.sendUnicastLoadQuery((String)session.getIdInternal(), "" + session.getVersion(), partnerInstanceName);
        } catch (Throwable th) {
            _logger.log(Level.INFO, "error during unicast load test to " + partnerInstanceName, th);
        }
        return resultState;
        //end unicast load test
    }
   
    public ReplicationState processUnicastfindsession(ReplicationState queryState) {
  return doFindSession(queryState, false);
    }

    /**
     * process the load of a Session
     * @param sessionState - contains the load command
     */
    public ReplicationState processLoadsession(ReplicationState sessionState) {
        //System.out.println("in processLoadsesson:sessionState = " + sessionState);
        ReplicationState result = processUnicastfindsession(sessionState);
        //System.out.println("in processLoadsession:result = " + result);
        return result;
    }
   
   //SJSAS 6406580 START
   
   /**
   * does the remove of session
   *
   * @param sessionId
   *   The session id to remove
   */   
    protected void doRemove(String sessionId) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationManagerBase",
                             "doRemove");
        }               
        ReplicationStore store = null;
        try {
            store = (ReplicationStore)getSessionStore();
            if(store != null) {
                store.doRemove(sessionId);
            }
        } catch (IOException ex) {
            if (_logger.isLoggable(Level.FINE)) {
                _logger.log(Level.FINE, "exception occurred in doRemove sessionId=" + sessionId, ex);
            }           
        } finally {
            this.putSessionStore(store);           
        }                
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationManagerBase",
                            "doRemove");
        }
    }    
   
    //SJSAS 6406580 END

   /**
   * remove sessions in the array of sessionIds from manager cache
   *
   * @param sessionIds
   *   The session ids to remove
   */    
    protected void removeSessionIdsFromManagerCache(String[] sessionIds) {
        synchronized(sessions) {
            for(int i=0; i<sessionIds.length; i++) {
                String nextId = sessionIds[i];
                // Take it out of the cache - remove does not handle nulls
                // TBD: Call super.remove instead?
                if(nextId != null) {
                    sessions.remove(nextId);
                }
            }
        }
    }

   /**
   * remove sessions from manager cache
   *
   * @param session
   *   The session to remove
   */
    public void removeSessionFromManagerCache(Session session) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>removeSessionFromManagerCache:session = " + session);
        }
        if(session == null) {
            return;
        }
        Session removed = null;
        // TBD: Call super.remove instead?
        synchronized (sessions) {
            removed = (Session) sessions.remove(session.getIdInternal());
        }
        if (removed != null && _salogger.isLoggable(Level.FINE)){
            _salogger.fine("Remove from manager cache id=" + session.getId());
        }
    }

    //Begin Expat List for Session
   
    public void handleDynamicOwnershipChanges(Event event,
                                              String triggeringInstance,
                boolean isLbStateChange) {
        if(!replicationUtil.isInstanceLoadBalancedByCLB() || !isConverged()) {
            return;
        }
  // If you need to perform work not related to the lb state,
  // do it here.
  if (!isLbStateChange) {
      return;
  }
  sessExpatListHandler.eventReceived(event, triggeringInstance);
  // Work only about the lb state (e.g. the expat list) is done here.
        if(!sessExpatListHandler.shouldCalculateExpatList()) {
            return; // expat calculation is already in progress.
        }
        long startTime = System.currentTimeMillis();
        boolean isExpatListLocked = false;
        try {
            ExpatListHandler.expatCalculationStarted();
            currentEvent = event;
            currentTriggeringInstance = triggeringInstance;
            synchronizeKeys();
            sessExpatListHandler.preInvoke(event, triggeringInstance);
            if (sessExpatListHandler.isAwaitingExpatList()) { // IMP :: this check can be done only after preInvoke
                // if we need to await for any list to be pushed to us from
                // remote instance, then only lock the list, otherwise allow
                // read on the expat list.
                expatReadWriteLock.writeLock().lock();
                isExpatListLocked = true;
            }
            getExpatList();
        } finally {
            expatIdsMap = mergeWithReplicaExpats(
                    expatIdsMap, sessReplicaExpatHandler);
            sessExpatListHandler.postInvoke();
            ExpatListHandler.expatCalculationEnded();
            if (isExpatListLocked) {
                expatReadWriteLock.writeLock().unlock();
            }
            currentEvent = null;
            currentTriggeringInstance = null;
            // Print the final map after the merge to check if everything is fine.
            ReplicationUtil.printExpatList(expatIdsMap, "HttpSession(final)", startTime);
        }
    }

    private Event currentEvent;
    private String currentTriggeringInstance;

    /**
     * Invoke the synchronizeKeys method asynchronously on the store(s).
     * Note that this method is common to all stores (in-house or third party)
     */
    private void synchronizeKeys() {
        // TODO :: For the third party implementation to work,
        // TODO :: we need to revisit the parameters.
        new SynchronizeKeysTask(getBackingStore(), null, null, false);
    }

    public void __synchronizeKeys(JxtaBackingStoreImpl jxtaBackingStore) {
        ExpatListHandler expatHandler = sessReplicaExpatHandler;
        if (expatHandler != null) {
            expatHandler.preInvoke(currentEvent, currentTriggeringInstance);
            expatHandler.call();
            // NOTE :: Since the result of the above call needs to be retained
            // until the results are merged with activeExpat, hence the invocation of
            // postInvoke is delayed until then. mergeWithReplicaExpats method
            // calls postInvoke after merging the replica expat with active expat.
        }
    }

    public void __createStoreExpatHandler(JxtaBackingStoreImpl jxtaBackingStore) {
        if (sessReplicaExpatHandler == null) {
            sessReplicaExpatHandler = ExpatListHandler.getExpatListHandler(
                    this, this.getApplicationId(),
                    MESSAGE_BROADCAST_EXPAT_QUERY_SESSION_REPLICA, MODE_WEB, "HttpSessionReplica");
        }
    }
   
    /**
     * For the third party backing store implementations,
     * this method will be a no-op.
     */
    private HashMap mergeWithReplicaExpats(HashMap activeExpat,
                                           ExpatListHandler replicaExpatHandler) {
        HashMap result = activeExpat;
        if(replicaExpatHandler != null) {
            try {
                HashMap replicaExpat = replicaExpatHandler.awaitAndGetResult();
                if(replicaExpat != null && !replicaExpat.isEmpty()) {
                    if(activeExpat != null) {
                        replicaExpat.putAll(activeExpat);
                    }
                    result = replicaExpat;
                }
            } finally {
                // since we have the results now, we can call postInvoke() and
                // clear the memory references.
                replicaExpatHandler.postInvoke();
            }
        }
        return result;
    }

   
   
    boolean isExpectingExpatIdsMap() {
        return sessExpatListHandler.isAwaitingExpatList();
    }
   
    public void getExpatList() {
        HashMap result = sessExpatListHandler.call();
        if(result != null) {
            expatIdsMap = result;
        }
    }
   
    public String getExpatPushCommandFor(String expatCommand) {
        if(MESSAGE_BROADCAST_EXPAT_QUERY_SESSION.equals(expatCommand)) {
            return MESSAGE_BROADCAST_EXPAT_RECEIVE_SESSION;
        }
        if(MESSAGE_BROADCAST_EXPAT_QUERY_SESSION_REPLICA.equals(expatCommand)) {
            return MESSAGE_BROADCAST_EXPAT_RECEIVE_SESSION_REPLICA;
        }
        throw new IllegalArgumentException("Invalid command " + expatCommand);
    }
   
    public HashMap<String,ExpatListElement> pruneExpatList(String expatCommand) {
        if (MESSAGE_BROADCAST_EXPAT_QUERY_SESSION.equals(expatCommand)) {
            // TODO :: how to prune?
            return expatIdsMap;
        }
        if (MESSAGE_BROADCAST_EXPAT_QUERY_SESSION_REPLICA.equals(expatCommand)) {
            return new HashMap<String, ExpatListElement>();
        }
        throw new IllegalArgumentException("Invalid command " + expatCommand);
    }

    public HashSet<ExpatListElement> getExpatListFor(String instanceName,
                                                     String command) {
        boolean isRemoteInstance = !ReplicationUtil.getInstanceName().
                equals(instanceName);
        if (MESSAGE_BROADCAST_EXPAT_QUERY_SESSION.equals(command)) {
            return isRemoteInstance ? getHttpSessionExpatIdsFromActive(instanceName)
                    : new HashSet<ExpatListElement>();
//            return getSessionExpatIds(instanceName, isRemoteInstance);
        }
        if(MESSAGE_BROADCAST_EXPAT_QUERY_SESSION_REPLICA.equals(command)) {
            return getHttpSessionExpatIdsFromReplica(instanceName);
        }
        throw new IllegalArgumentException("Invalid command " + command);
    }

    public void getExpatLists(String expatCommand, ExpatListQueryResults result) {
        if(MESSAGE_BROADCAST_EXPAT_QUERY_SESSION.equals(expatCommand)) {
            getHttpSessionExpatIdsFromActive(result);
//            getAllSessionExpatIds(result);
        } else if(MESSAGE_BROADCAST_EXPAT_QUERY_SESSION_REPLICA.equals(expatCommand)) {
            getHttpSessionExpatIdsFromReplica(result);
        } else {
            throw new IllegalArgumentException("Invalid command " + expatCommand);
        }
    }

    public void processBroadcastreceivesessionexpatids(
            ReplicationState expatListState) {
        ExpatListListener expatListener =
                sessExpatListHandler.getListener();
        if(expatListener != null) {
            expatListener.expatListReceived(expatListState);
        }
    }

    private void expatListReceived(ReplicationState expatListState,
                                                 ExpatListHandler expatHandler) {
        if(expatHandler != null) {
            ExpatListListener expatListener =
                    expatHandler.getListener();
            if (expatListener != null) {
                expatListener.expatListReceived(expatListState);
            }
        }
    }

    public void processBroadcastreceivesessionreplicaexpatids(
            ReplicationState expatListState) {
        expatListReceived(expatListState, sessReplicaExpatHandler);
    }
   
    /**
     * Get the HttpSession' expat list from the active cache for a given instance.
     */
    private HashSet<ExpatListElement> getHttpSessionExpatIdsFromActive(
            String requestingInstance) {
        //using set to avoid dups
        HashSet expatIdsSet = new HashSet();
        long version = -1L;
        //iterate over active sessions cache
        //take copy asap to avoid synchronizing too long
        List sessionsCopiedList = getClonedSessionsList();
        Iterator it2 = sessionsCopiedList.iterator();
        while (it2.hasNext()) {
            BaseHASession sess = (BaseHASession) it2.next();
            String sessId = (String) sess.getIdInternal();
            //use session beKey for mapping to owning instance
            String rightfulOwner = replicationUtil.
                    getActualServerInstanceForBeKey(sess.getBeKey());
            if (requestingInstance.equalsIgnoreCase(rightfulOwner)) {
                version = sess.getVersion();
                expatIdsSet.add(new ExpatListElement(sessId, version,
                        ReplicationUtil.getInstanceName(), true));
            }
        }
        return expatIdsSet;
    }

    /**
     * Get the HttpSession' expat list from the replica cache for a given instance.
     */
    private HashSet<ExpatListElement> getHttpSessionExpatIdsFromReplica(
            String requestingInstance) {
        //using set to avoid dups
        HashSet expatIdsSet = new HashSet();
        //iterate over session replicas
        BaseCache replicatedSessionsCache = getReplicatedSessions();
        Iterator it = replicatedSessionsCache.values();
        long version = -1L;
        while (it.hasNext()) {
            ReplicationState state = (ReplicationState) it.next();
            String sId = (String) state.getId();
            //use bekey for mapping instance ownership
            String beKey = (String) state.getProperty(BEKEY);
            if (beKey != null) {
                String rightfulOwner = replicationUtil.
                        getActualServerInstanceForBeKey(beKey);
                if (requestingInstance.equalsIgnoreCase(rightfulOwner)) {
                    version = state.getVersion();
                    expatIdsSet.add(new ExpatListElement(sId, version,
                            ReplicationUtil.getInstanceName()));
                }
            }
        }
        return expatIdsSet;
    }

    /**
     * Get the HttpSession' expat list from the active cache for all the surviving instances.
     */
    private void getHttpSessionExpatIdsFromActive(ExpatListQueryResults results) {
        //iterate over session replicas
        long version = -1L;
        //iterate over active sessions cache
        //take copy asap to avoid synchronizing too long
        List sessionsCopiedList = getClonedSessionsList();
        Iterator it2 = sessionsCopiedList.iterator();
        while (it2.hasNext()) {
            BaseHASession sess = (BaseHASession) it2.next();
            String ssId = (String) sess.getIdInternal();
            //use beKey from session for mapping to owning instance
            String rightfulOwner = replicationUtil.getActualServerInstanceForBeKey(
                    sess.getBeKey());
        }
        //now iterator over active sessions copied list
        Iterator it3 = sessionsCopiedList.iterator();
        while (it3.hasNext()) {
            BaseHASession sess = (BaseHASession) it3.next();
            String ssId = (String) sess.getIdInternal();
            //use beKey from session for mapping to owning instance
            String rightfulOwner = replicationUtil.getActualServerInstanceForBeKey(
                    sess.getBeKey());
            if (rightfulOwner != null && !rightfulOwner.equalsIgnoreCase(
                    ReplicationUtil.getInstanceName())) {
                version = sess.getVersion();
                ExpatListElement expatElement = new ExpatListElement(ssId,
                        version, ReplicationUtil.getInstanceName());
                results.addQueryResultFor(rightfulOwner, expatElement);
            }
        }
    }

    /**
     * Get the HttpSession' expat list from the replica cache for all the surviving instances.
     */
    private void getHttpSessionExpatIdsFromReplica(ExpatListQueryResults results) {
        //iterate over session replicas
        BaseCache replicatedSessionsCache = getReplicatedSessions();
        Iterator it = replicatedSessionsCache.values();
        long version = -1L;
        while (it.hasNext()) {
            ReplicationState state = (ReplicationState) it.next();
            String ssId = (String) state.getId();
            //use bekey for mapping instance ownership
            String beKey = (String) state.getProperty(BEKEY);
            if (beKey != null) {
                String rightfulOwner = replicationUtil.
                        getActualServerInstanceForBeKey(beKey);
                version = state.getVersion();
                ExpatListElement expatElement = new ExpatListElement(ssId,
                        version, ReplicationUtil.getInstanceName());
                results.addQueryResultFor(rightfulOwner, expatElement);
            }
        }
    }

    /**
     * Gets the Session expatIds for a given instance
     *
     * @param requestingInstance the querying instance for
     *  which we are obtaining a list
     * @param consultActiveCache should the list consider
     *  Sessions in the active cache
     *
     * @return The List of expatId elements
     */
/*
    private HashSet<ExpatListElement> getSessionExpatIds(
            String requestingInstance, boolean consultActiveCache) {
        //using set to avoid dups
        HashSet expatIdsSet = new HashSet();
        //iterate over session replicas
        BaseCache replicatedSessionsCache = getReplicatedSessions();
        Iterator it = replicatedSessionsCache.values();
        long version = -1L;
        while (it.hasNext()) {
            ReplicationState state = (ReplicationState) it.next();
            String sId = (String) state.getId();
            //use bekey for mapping instance ownership
            String beKey = (String) state.getProperty(BEKEY);
            if (beKey != null) {
                String rightfulOwner = replicationUtil.
                        getActualServerInstanceForBeKey(beKey);
                if (requestingInstance.equalsIgnoreCase(rightfulOwner)) {
                    version = state.getVersion();
                    expatIdsSet.add(new ExpatListElement(sId, version,
                            ReplicationUtil.getInstanceName()));
                }
            }
        }
        if (consultActiveCache) {
            //iterate over active sessions cache
            //take copy asap to avoid synchronizing too long
            List sessionsCopiedList = getClonedSessionsList();
            Iterator it2 = sessionsCopiedList.iterator();
            while (it2.hasNext()) {
                BaseHASession sess = (BaseHASession) it2.next();
                String sessId = (String) sess.getIdInternal();
                //use session beKey for mapping to owning instance
                String rightfulOwner = replicationUtil.
                        getActualServerInstanceForBeKey(sess.getBeKey());
                if (requestingInstance.equalsIgnoreCase(rightfulOwner)) {
                    version = sess.getVersion();
                    expatIdsSet.add(new ExpatListElement(sessId, version,
                            ReplicationUtil.getInstanceName(), true));
                }
            }
        }
        return expatIdsSet;
    }

    private void getAllSessionExpatIds(ExpatListQueryResults results) {
        //iterate over session replicas
        BaseCache replicatedSessionsCache = getReplicatedSessions();
        Iterator it = replicatedSessionsCache.values();
        long version = -1L;
        while (it.hasNext()) {
            ReplicationState state = (ReplicationState) it.next();
            String ssId = (String) state.getId();
            //use bekey for mapping instance ownership
            String beKey = (String) state.getProperty(BEKEY);
            if (beKey != null) {
                String rightfulOwner = replicationUtil.
                        getActualServerInstanceForBeKey(beKey);
                version = state.getVersion();
                ExpatListElement expatElement = new ExpatListElement(ssId,
                        version, ReplicationUtil.getInstanceName());
                results.addQueryResultFor(rightfulOwner, expatElement);
            }
        }
        //iterate over active sessions cache
        //take copy asap to avoid synchronizing too long
        List sessionsCopiedList = getClonedSessionsList();
        Iterator it2 = sessionsCopiedList.iterator();
        while (it2.hasNext()) {
            BaseHASession sess = (BaseHASession) it2.next();
            String ssId = (String) sess.getIdInternal();
            //use beKey from session for mapping to owning instance
            String rightfulOwner = replicationUtil.getActualServerInstanceForBeKey(
                    sess.getBeKey());
        }
        //now iterator over active sessions copied list
        Iterator it3 = sessionsCopiedList.iterator();
        while (it3.hasNext()) {
            BaseHASession sess = (BaseHASession) it3.next();
            String ssId = (String) sess.getIdInternal();
            //use beKey from session for mapping to owning instance
            String rightfulOwner = replicationUtil.getActualServerInstanceForBeKey(
                    sess.getBeKey());
            if (rightfulOwner != null && !rightfulOwner.equalsIgnoreCase(
                    ReplicationUtil.getInstanceName())) {
                version = sess.getVersion();
                ExpatListElement expatElement = new ExpatListElement(ssId,
                        version, ReplicationUtil.getInstanceName());
                results.addQueryResultFor(rightfulOwner, expatElement);
            }
        }
    }

    private List getSessionExpatIdsThirdPartySPI(String owningInstanceName) {
        BackingStore backingStore
            = this.getBackingStore();
        HttpSessionExtraParams extraParamCriteria
            = HttpSessionExtraParams.createSearchCriteria(
                owningInstanceName);
        Collection<HttpSessionExtraParams> sessColl
            = backingStore.findByCriteria(extraParamCriteria);

        List expatIds = new ArrayList();
        //using set to avoid dups
        HashSet expatIdsSet = new HashSet();
        Iterator<HttpSessionExtraParams> dfResults =
            (Iterator<HttpSessionExtraParams>) sessColl.iterator();
        while (dfResults.hasNext()) {
            HttpSessionExtraParams eParam = dfResults.next();
            if (owningInstanceName.equals(eParam.getCurrentOwnerInstanceName())) {
                expatIdsSet.add(new ExpatListElement((String)eParam.getId(), -1L, null));
            }
        }
        long nextVersion = -1L;
        //iterate over active sessions cache
        //take copy asap to avoid synchronizing too long
        List sessionsCopiedList = getClonedSessionsList();
        Iterator it2 = sessionsCopiedList.iterator();
        while(it2.hasNext()) {
            BaseHASession nextSession
                = (BaseHASession)it2.next();
            String nextSessionId
                = (String)nextSession.getIdInternal();
            if(nextSession.getExtraParameters() != null) {
                //use session beKey for mapping to owning instance
                String nextRightfulOwnerInstanceName
                    = replicationUtil.getActualServerInstanceForBeKey(nextSession.getBeKey());
                if(nextRightfulOwnerInstanceName != null
                    && nextRightfulOwnerInstanceName.equalsIgnoreCase(owningInstanceName)) {
                    nextVersion = nextSession.getVersion();
                    expatIdsSet.add(new ExpatListElement(nextSessionId, nextVersion, ReplicationUtil.getInstanceName(), true));
                }
            }
        }
        expatIds.addAll(expatIdsSet);
        return expatIds;
    }
*/

    private List getClonedSessionsList() {
        //take copy asap to avoid synchronizing too long
        List sessionsCopiedList = new ArrayList();
        synchronized(sessions) {
            Iterator it2 = sessions.values().iterator();
            while (it2.hasNext()) {
                sessionsCopiedList.add(it2.next());
            }
        }
        return sessionsCopiedList;
    }

    /**
     * process the processBroadcastfindsessionexpatids for SipSession
     * @param queryState
     */
    public ReplicationState processBroadcastfindsessionexpatids(ReplicationState queryState) {
        //complete query and send back response
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindsessionexpatids:instance: " + ReplicationUtil.getInstanceName());
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindsessionexpatids:owningInstance=" + queryState.getExtraParam());
        }
  if(replicationUtil.isInstanceLoadBalancedByCLB() &&
                !ReplicationHealthChecker.isServerLbEnabled(
                        ReplicationUtil.getInstanceName())) {
            // Don't respond if the CLB is running but we are not CLB enabled
            return null;
        }
        sessExpatListHandler.sendExpatQueryResponse(queryState);
        return null;
    }

    public ReplicationState processBroadcastfindsessionreplicaexpatids(ReplicationState queryState) {
        //complete query and send back response
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindsessionreplicaexpatids:instance: " + ReplicationUtil.getInstanceName());
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindsessionreplicaexpatids:owningInstance=" + queryState.getExtraParam());
        }
  if(replicationUtil.isInstanceLoadBalancedByCLB() &&
                !ReplicationHealthChecker.isServerLbEnabled(
                        ReplicationUtil.getInstanceName())) {
            // Don't respond if the CLB is running but we are not CLB enabled
            return null;
        }
        if(sessExpatListHandler != null) {
            sessExpatListHandler.sendExpatQueryResponse(queryState);
        }
        return null;
    }

    //End Expat List for Session
   
    //begin processing methods
   
    public void processSave(ReplicationState message) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN" + this.getClass().getName() + ">>processSave");
        }       
        this.putInReplicationCache(message);
    }
   
    public ReplicationState processSize(ReplicationState message) {
        BaseCache replicatedSessionsCache = getReplicatedSessions();
        int result = replicatedSessionsCache.getEntryCount();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("processSize: entryCount=" + result);
        }        
        ReplicationState resultState
            = ReplicationState.createQueryStateResponse(MODE_WEB, message.getAppId(), message.getAppId(), message.getInstanceName(), Integer.valueOf(result));
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("processSize:resultState=" + resultState);
        }       
        return resultState;
    }   
   
    public void processValvesave(ReplicationState message) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN" + this.getClass().getName() + ">>processValvesave");
            _logger.fine("processValvesave:id:" + message.getId());
            _logger.fine("processValvesave:version:" + message.getVersion());
        }
        this.putInReplicationCache(message);
    }
   
    public void processCompositesave(ReplicationState message) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN" + this.getClass().getName() + ">>processCompositesave");
            _logger.fine("processCompositesave:version:" + message.getVersion());
        }
        ReplicationState currentState = null;
        Object monitor = getReplicationSessionMonitor((String)message.getId());
        synchronized(monitor) {        
            currentState
                = this.getFromReplicationCache((String)message.getId());
            //if we do not have it in cache, just put it in
            if(currentState == null) {
                this.putInReplicationCache(message);
                return;
            }
            //otherwise update with deltas
            ReplicationState updatedState = this.updateReplicationState(currentState, message);
            this.putInReplicationCache(updatedState);
        }
    }
   
    private ReplicationState updateReplicationState(ReplicationState currentState, ReplicationState newState) {
        CompositeMetadata currentComposite
            = ReplicationState.createCompositeMetadataFrom(currentState);
        CompositeMetadata newComposite
            = ReplicationState.createCompositeMetadataFrom(newState);
        CompositeMetadata resultComposite
            = this.applyCompositeMetadataDeltas(currentComposite, newComposite);
        byte[] resultAttributeState = null;
        try {           
            resultAttributeState
                = ReplicationState.getByteArrayFromCollection(resultComposite.getEntries());
        } catch (IOException ex) {}
        ReplicationState updatedState
            = ReplicationState.createUpdatedStateFrom(newState, resultAttributeState);
        return updatedState;
    }
   
    private CompositeMetadata applyCompositeMetadataDeltas(CompositeMetadata current, CompositeMetadata deltas) {
        Collection<SessionAttributeMetadata> currentCollection = current.getEntries();
        Collection<SessionAttributeMetadata> deltasCollection = deltas.getEntries();
        Iterator deltasIterator = deltasCollection.iterator();
        while(deltasIterator.hasNext()) {
            SessionAttributeMetadata nextAttributeMetadata
                = (SessionAttributeMetadata)deltasIterator.next();
            this.applyCompositeMetadataDelta(nextAttributeMetadata, currentCollection);
        }
        return current;
    }
   
    private void applyCompositeMetadataDelta(SessionAttributeMetadata nextAttributeMetadata, Collection<SessionAttributeMetadata> currentCollection) {
        switch(nextAttributeMetadata.getOperation()) {
            case DELETE: currentCollection.remove(nextAttributeMetadata);
            case ADD: currentCollection.add(nextAttributeMetadata);
            case UPDATE:
                if(currentCollection.contains(nextAttributeMetadata)) {
                    currentCollection.remove(nextAttributeMetadata);
                }
                currentCollection.add(nextAttributeMetadata);
                break;
        }
    }

    /**
     * process the updateLastAccessTime of the replicationState
     * @param message
     */   
    public void processUpdatelastaccesstime(ReplicationState message) {
        replicaCache.processUpdatelastaccesstime(message);
    }  
   
    public void processRemove(ReplicationState message) {
        if(message.getState() != null) {
            processBulkRemove(message);
            return;
        }
        replicaCache.processRemove(message);
    }
   
    public void processBulkRemove(ReplicationState message) {
        replicaCache.removeFromReplicationCache(message.getState());
    }

    public void processRemoveids(ReplicationState message) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN" + this.getClass().getName() + ">>processRemoveids");
        }
        BaseCache replicatedSessionsCache = getReplicatedSessions();
        //state of this message contains serialized list of ids to remove
        byte[] idsToRemoveState = message.getState();
        List removedIdsList = new ArrayList();
        try {
            removedIdsList = (List)ReplicationState.getObjectValue(idsToRemoveState);
        } catch (Exception ex) {
            //deliberately do nothing
        }
        //ReplicationState.displayStringList(removedIdsList);
        for(int i=0; i<removedIdsList.size(); i++) {
            String nextIdToRemove = (String)removedIdsList.get(i);
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine(">>processRemoveids:nextIdToRemove=" + nextIdToRemove);
            }
            replicatedSessionsCache.remove(nextIdToRemove);
            }
        }
   
    //this message sent from sso valve
    public void processAssociate(ReplicationState message) {
        //FIXME have moved this to have this processed in
        //web container for session
        //when session is loaded from replica need to apply
        //the ssoId from replica - same with lastAccess
        ReplicationState storedReplica
            = this.getFromReplicationCache((String)message.getId());
        //storedReplica.setSsoId(message.getSsoId()); ssoid is in extraParam
        storedReplica.setExtraParam(message.getExtraParam());
    }
   
    public void processMessage(ReplicationState message) {
        //handle broadcast methods
        if(ReplicationState.isBroadcastState(message)) {
            processBroadcastMessage(message);
            return;
        }
       
        //handle non-void methods
        ReplicationStateQueryResponse queryResult = null;
        //do process non-void message (and cannot be response either)
        if(!message.isResponseState() && !message.isVoidMethodReturnState()) {           
            //do non-void processing including sending response
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("ReplicationManagerBase>>processMessage -sending response:command: " + message.getCommand());             
            }           
            queryResult = this.doProcessQueryMessage(message);
            ReplicationState qResponse = queryResult.getState();
            if(qResponse != null) {
                //sourceInstanceName is preserved in the response
                ReplicationState response =
                    ReplicationState.createResponseFrom(qResponse);
                if(_logger.isLoggable(Level.FINE)) {
                    _logger.fine("RepMgrBase:responseState=" + response);
                }               
                this.doSendResponse(response);
            }
            return;
        }
        //end do process non-void message
       
        /*
        ReplicationState response =
            ReplicationState.createResponseFrom(message);
         */
        //send a response before further processing only if processed
        //msg is not itself a response and if method is a void return type
        //FIXME this send will be removed if upstream ack send works
        /* removing this for test
        if(!message.isReturnMessage() && message.isVoidMethodReturnState()) {
            this.doSendResponse(response);
        }
        */       

        boolean isResponse = this.doProcessMessage(message);
       
        //send a response only if processed msg is not itself a response
        //and if method is not void return type (in that case ack was
        //already sent)
        /*
        if(!isResponse && !message.isVoidMethodReturnState()) {
            //ReplicationState response =
            //    ReplicationState.createResponseFrom(message);
            this.doSendResponse(response);
        }
         */
    }
   
    /**
    * send the response
    *
    * @param sessionState
    *   The replication state response
    */   
    public void doSendResponse(ReplicationState sessionState) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationManagerBase",
                             "doSendResponse");
        }               
        ReplicationStore store = null;
        try {
            store = (ReplicationStore)getSessionStore();
            if(store != null) {
                store.sendResponse(sessionState);
            }
        } catch (Exception ex) {
            if (_logger.isLoggable(Level.FINE)) {
                _logger.log(Level.FINE, "exception occurred in doSendResponse id=" + sessionState.getId(), ex);
            }
        } finally {
            this.putSessionStore(store);           
        }                
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationManagerBase",
                            "doSendResponse");
        }
    } 
   
    //return true if message is processResponse
    public boolean doProcessMessage(ReplicationState message) {
        boolean result = false;
        String methodName = ReplicationUtil.getProcessMethodName(message);
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>doProcessMessageName=" + methodName);
        }       
        try {
            Class myClass = this.getClass();
            myClass.getMethod(
                methodName,
                    new Class[]{ message.getClass() }).invoke(
                        this, new Object[]{ message });
        } catch (Throwable t) {
           _logger.log(
               Level.SEVERE,
               "Unable to process replication message with methodName=" +
               methodName, t);
        }
        if(methodName.equals("processResponse")) {
            result = true;
        }
        return result;
    }
   
    //return true if message is processQueryResponse
    public ReplicationStateQueryResponse doProcessQueryMessage(ReplicationState message) {      
        ReplicationState resultState = null;
        String methodName = ReplicationUtil.getProcessMethodName(message);
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>doProcessQueryMessage:methodName=" + methodName);
            _logger.fine("in " + this.getClass().getName() + ">>doProcessQueryMessage:thisInstance=" + ReplicationUtil.getInstanceName() + "SASEreturnInstance=" + message.getInstanceName() );
        }        
        try {
            Class myClass = this.getClass();
            resultState = (ReplicationState) myClass.getMethod(
                methodName,
                    new Class[]{ message.getClass() }).invoke(
                        this, new Object[]{ message });
        } catch (Throwable t) {
           _logger.log(
               Level.SEVERE,
               "Unable to process replication message with methodName=" +
               methodName, t);
        }
        boolean isResponse = (methodName.equals("processBroadcastresponse") || methodName.equals("processUnicastresponse"));
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>doProcessQueryMessage:resultState=" + resultState);
        }
        if (resultState != null) {
            resultState.setRouteAdvertisement(message.getRouteAdvertisement());
        }       
        return new ReplicationStateQueryResponse(resultState, isResponse);
    }
   
    public void processBroadcastMessage(ReplicationState message) {
        if (_salogger.isLoggable(Level.FINER) && message != null) {
            _salogger.finer("processBroadcastmessage cmd=" + message.getCommand() + " appid=" + message.getAppId() + " id=" + message.getId() +
                            " version=" + message.getVersion());
        }

        ReplicationStateQueryResponse response = this.doProcessQueryMessage(message);
        boolean isResponse = response.isResponse();
        ReplicationState responseState = response.getState();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("processBroadcastMessage:after doProcessQueryMessage:response=" + isResponse);
            _logger.fine("processBroadcastMessage:after doProcessQueryMessage:responseState=" + responseState);
            _logger.fine("processBroadcastMessage:after doProcessQueryMessage:responseStateTrunk=" + responseState.getTrunkState());
            _logger.fine("processBroadcastMessage:after doProcessQueryMessage:responseStateAttr=" + responseState.getState());
            _logger.fine("processBroadcastMessage:after doProcessQueryMessage:responseStateVer=" + responseState.getVersion());
        }
      
        //don't send a response to a response
        if(!isResponse) {
            //point-to-point response back to sender
            //doSendQueryResponse(responseState, this.getInstanceName());
            doSendQueryResponse(responseState, message.getInstanceName());
            if (responseState != null) {
                saloggerQueryResponse(responseState, message.getInstanceName());
            }
        }
    }
   
    public void processQueryMessage(ReplicationState message, String returnInstance) {
        if (_salogger.isLoggable(Level.FINER) && message != null) {
            _salogger.finer("processQueryMessage cmd=" + message.getCommand() + " appid=" + message.getAppId() + " id=" + message.getId() +
                            " version=" + message.getVersion() + " returnToInstance=" + returnInstance);
        }
        ReplicationStateQueryResponse response = this.doProcessQueryMessage(message);
        boolean isResponse = response.isResponse();
        ReplicationState responseState = response.getState();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("processQueryMessage:after doProcessQueryMessage:response=" + isResponse);
            _logger.fine("processQueryMessage:after doProcessQueryMessage:responseState=" + responseState);           
        }       
        //don't send a response to a response
        if(!isResponse && responseState != null) {
            //point-to-point response back to sender
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("processQueryMessage - need to send back result to " + returnInstance);           
            }
            doSendQueryResponse(responseState, returnInstance);
            if (responseState != null) {
                saloggerQueryResponse(responseState, returnInstance);
            }
        }
    }
   
    /**
     * Log negative acknowledgement of query at FINER level.
     * Log positive acknowledgement of query at FINE level.
     */
    private void saloggerQueryResponse(ReplicationState responseState, String returnInstance) {
        final Level LEVEL = responseState.isNack() ? Level.FINER : Level.FINE;
        final String RESPONSE = responseState.isNack() ? "negative ack" : "replica";
        if (_salogger.isLoggable(LEVEL)) {
            _salogger.log(LEVEL, "sent " + RESPONSE + " for replication query response " +
                    " mode=" + responseState.getMode() +
                    " cmd=" + responseState.getCommand() +
                    " appId=" + responseState.getAppId() +
                    " id=" + responseState.getId() +
                    "[vers:" + responseState.getVersion() + "]" +
                    " instance:" + returnInstance);
        }
    }
   
    /**
    * send the response
    *
    * @param sessionState
    *   The replication state response
    * @param instanceName  the name of the target instance
    */   
    public void doSendQueryResponse(ReplicationState sessionState, String instanceName) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationManagerBase",
                             "doSendQueryResponse");
        }       
        JxtaReplicationSender replicationSender =
                JxtaReplicationSender.createInstance();
        ReplicationState resultState =
                replicationSender.sendReplicationStateQueryResponse(sessionState, instanceName, useReplicationUnicastLoadResponseBatching);                       
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationManagerBase",
                            "doSendQueryResponse");
        }
    }
       
    public void processResponse(ReplicationState message) {
        //complete processing response - not sending response to a response
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN" + this.getClass().getName() + ">>processResponse");           
        }       
        ReplicationResponseRepository.putEntry(message);
    }
   
    public ReplicationState processBroadcastresponse(ReplicationState queryResponseState) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastresponse:queryResponseState=" + queryResponseState);           
        }
        //ReplicationResponseRepository.putEntry(queryResponseState);
        ReplicationResponseRepository.putFederatedEntry(queryResponseState);
        return queryResponseState;
    }

    public int processExpiredSessions() {
        processExpiredReplicas();
        return 0;
    }
   
    void processExpiredReplicas() {
        replicaCache.processExpiredReplicas();
        //check and remove stale replica monitors
        replicatedSessionMonitors.processExpired();
    }

    public ReplicationState processBroadcastfindsession(ReplicationState queryState) {
  return doFindSession(queryState, true);
    }

    public ReplicableEntity findSessionAndPassivate(String id,
                                                    AtomicBoolean cachedSessionIsLocal) {
        try {
            return (BaseHASession) this.findSessionFromCacheOnly(id);
        } catch (IOException ex) {
            return null;
        }
        // TODO :: is checking cachedSessionIsLocal and passivation required for converged sessions?
    }

    private ReplicationState doFindSession(ReplicationState queryState, boolean isBroadcast) {
        //complete query and send back response
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindSession:instance: " + ReplicationUtil.getInstanceName());
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindSession:id=" + queryState.getId());
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindSession:version=" + queryState.getVersion());
        }
        //printReplicatedSessionIds();
        ReplicationState returnState = null;
        BaseHASession sess = null;
        //first look in active cache
        try {
            sess = (BaseHASession)this.findSessionFromCacheOnly((String)queryState.getId());
        } catch (IOException ex) {}

        //if queryVersion == -1, then this is version unaware load
        long queryVersion = queryState.getVersion();
        ReplicationState replicaState = null;
        Object monitor = getReplicationSessionMonitor((String)queryState.getId());
        synchronized(monitor) {
            replicaState = findReplicatedState(queryState);
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("in " + this.getClass().getName() + ">>findReplicatedState result=" + replicaState);
            }
            returnState
                = getBestSession(sess, replicaState, queryState, isBroadcast);
        }
        //at this point we know replicaState is not null
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("processBroadcastfindsession:REPLICA_FOUND:replicaStateVersion:" + returnState.getVersion());
            _logger.fine("processBroadcastfindsession:REPLICA_FOUND:replicaState:" + returnState.getTrunkState());
            _logger.fine("processBroadcastfindsession:REPLICA_FOUND:replicaAttrState" + returnState.getState());
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindSession:returnState=" + returnState);
        }
        return returnState;

    }

    /**
     * find the best version of Session
     * removing any stale versions and return query result
     * @param activeSession Session
     *      from active cache
     * @param replicaSession Session
     *      from replica cache
     * @param queryState version requested in query (-1 means
     *      version unaware
     */
    private ReplicationState getBestSession(BaseHASession activeSession,
        ReplicationState replicaSession, ReplicationState queryState,
          boolean isBroadcast) {
        ReplicationState bestResult = null;
        long queryVersion = queryState.getVersion();

        //first check for none found in either active or replica caches
        if(replicaSession == null && activeSession == null) {
            //return nack
            return ReplicationState.createQueryResponseFrom(queryState, true);
        }

        //next check for artifacts found in both active and replica caches
        if(replicaSession != null && activeSession != null) {
            //next check for stale version if version-aware query
            //compare and remove the lesser version
            //keeping the higher version as (tentative) best
            if(queryVersion != -1 &&
                replicaSession.getVersion() < queryVersion && !isRelaxCacheVersionSemantics()) {
                //remove stale replica - work with active
                removeFromReplicationCache((String)replicaSession.getId());
                //create appropriate response from active
                bestResult
                    = createSessionResponseFrom(activeSession, queryState, isBroadcast);
            } else {
                //remove stale active - work with replica
                clearFromManagerCache(activeSession.getId());
                //create appropriate response from replica
                bestResult
                    = createSessionResponseFrom(replicaSession, queryState);
            }
        } else {
            //either replica or active is null and other is non-null
            //replica is null and active is not null
            if(replicaSession == null) {
                //create appropriate response from active
                bestResult
                    = createSessionResponseFrom(activeSession, queryState, isBroadcast);
            } else {
                //active is null & replica is not null
                //create appropriate response from replica
                bestResult
                    = createSessionResponseFrom(replicaSession, queryState);
            }
        }
        return bestResult;
    }

    private ReplicationState createSessionResponseFrom(ReplicationState replicaSession,
            ReplicationState queryState) {
        //create appropriate response from replica
        ReplicationState result = null;
        long queryVersion = queryState.getVersion();
        if(queryVersion != -1 && replicaSession.getVersion() < queryVersion) {
            //return nack & clear stale replica
            removeFromReplicationCache((String)replicaSession.getId());
            result = ReplicationState.createQueryResponseFrom(queryState, true);
        } else {
            //return real response based on replica
            result = ReplicationState.createQueryResponseFrom(replicaSession);
        }
        return result;
    }

    private ReplicationState createSessionResponseFrom(BaseHASession activeSession,
            ReplicationState queryState, boolean isBroadcast) {
        //create appropriate response from active
        ReplicationState result = null;
        long queryVersion = queryState.getVersion();
        if(queryVersion != -1 && activeSession.getVersion() < queryVersion) {
            //return nack & clear stale active
            clearFromManagerCache(activeSession.getIdInternal());
            result = ReplicationState.createQueryResponseFrom(queryState, true);
        } else {
            //return real response based on active Session
            try {
                result = createQueryResponseFrom(activeSession, isBroadcast);
            } catch (IOException ioe) {
                _logger.log(Level.WARNING,
                    "Failed load: Unable to serialize " +
                    activeSession, ioe);
                // We've been unable to serialize the given active
                // Session.
                // Clear it from the active cache and return a nack instead
                clearFromManagerCache(
                    activeSession.getIdInternal());
                result = ReplicationState.createQueryResponseFrom(queryState,
                                                                  true);
            }
        }
        return result;
    }

    /**
     * Converts the given BaseHASession to a ReplicationState.
     *
     * @param sess The BaseHASession to be converted
     *
     * @return The ReplicationState corresponding to the given
     * BaseHASession
     */
    private ReplicationState createQueryResponseFrom(
            BaseHASession sess, boolean isBroadcast) throws IOException {
        byte[] containerExtraParamState = null;
        HttpSessionExtraParams containerExtraParams
            = sess.getExtraParameters();
        if(containerExtraParams != null) {
            try {
                containerExtraParamState
                    = ReplicationUtil.getByteArray(containerExtraParams, isReplicationCompressionEnabled());
            } catch (IOException ex) {
                ; //deliberate no-op
            }
        }
        return new ReplicationState(
                MODE_WEB,
    (isBroadcast) ?
                    sess.getIdInternal() :
        LOAD_SESSION_COMMAND + ":" + sess.getIdInternal(),
                getApplicationId(),
                sess.getVersion(),
                0L,
                0L,
                null,
                null,
                null,
    (isBroadcast) ?
                    ReplicationState.RETURN_BROADCAST_MSG_COMMAND :
                    ReplicationState.RETURN_UNICAST_MSG_COMMAND,
                ReplicationUtil.getByteArray(sess, isReplicationCompressionEnabled()),
                null,
                containerExtraParamState);
    }

   
    public void processBroadcastloadreceived(ReplicationState queryState) {
        //load is acknowledged safe to remove replica now
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processbroadcastloadreceived:instance: " + ReplicationUtil.getInstanceName());
            _logger.fine("in " + this.getClass().getName() + ">>processbroadcastloadreceived:id=" + queryState.getId());
        }
        if(queryState == null || queryState.getId() == null) {
            return;
        }

        String ignoreInstance = (String) queryState.getProperty(ReplicationState.IGNORE_REMOVE_INSTANCE_NAME);
        //only safe to remove replica if we are not the replica partner
        //now determined by passed property
        if(ignoreInstance != null && !ignoreInstance.equals(ReplicationUtil.getInstanceName())) {
            removeFromReplicationCache((String)queryState.getId());
        }
    }          
   
    //end processing methods
   
    /**
    * send the response
    * //FIXME this should get removed after testing
     * we'll be using ReplicationState not converting from Session
    * @param session
    *   The session
    * @return ReplicationState
    */   
    public ReplicationState createQueryResponse(Session session) {   
        String command = ReplicationState.RETURN_BROADCAST_MSG_COMMAND;
        ReplicationStore store = (ReplicationStore)this.getStore();
        ReplicationState transmitState = null;
        try {
            transmitState = store.createReplicationState(session, command);
        } catch (IOException ex) {}
        return transmitState;
    }       
   
    protected ReplicationState findReplicatedState(ReplicationState queryState) {
        return replicaCache.findReplicatedState(queryState);
    }
   
    protected void clearFromManagerCache(String id) {
        Session sess = null;
        try {
            sess = this.findSessionFromCacheOnly(id);
        } catch (IOException ex) {}
        if(sess != null) {                              
            this.removeSessionFromManagerCache(sess);
        }
    }
   
   /**
    * Finds and returns the session with the given id that also satisfies
    * the given version requirement.
    *
    * This overloaded version of findSession() will be invoked only if
    * isSessionVersioningSupported() returns true. By default, this method
    * delegates to the version of findSession() that does not take any
    * session version number.
    *
    * @param id The session id to match
    * @param version The session version requirement to satisfy
    *
    * @return The session that matches the given id and also satisfies the
    * given version requirement, or null if no such session could be found
    * by this session manager
    *
    * @exception IOException if an IO error occurred
    */
    public Session findSession(String id, String version) throws IOException {
  Object monitor = getReplicationSessionMonitor(id);
  synchronized (monitor) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("in findSession: version=" + version);
            }
            if(!this.isSessionIdValid(id) || version == null) {
                return null;
            }
            Session session = null;
            long requiredVersion = 0L;
            long cachedVersion = -1L;
            try {
                requiredVersion = (Long.valueOf(version)).longValue();
            } catch (NumberFormatException ex) {
                //deliberately do nothing
            }
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("findSession:requiredVersion=" + requiredVersion);
            }               
            Session cachedSession = this.findSessionFromCacheOnly(id);
            if(cachedSession != null) {
                cachedVersion = cachedSession.getVersion();
            }
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("findSession:cachedVersion=" + cachedVersion);
            }        
            //if version match return cached session else purge it from cache
            //if relaxCacheVersionSemantics is set true then we return the
            //cached version even if it is greater than the required version
            if(cachedVersion == requiredVersion || (isRelaxCacheVersionSemantics() && (cachedVersion > requiredVersion))) {
                return cachedSession;
            } else {
                //if relaxCacheVersionSemantics - we do not remove because even
                //though stale we might return it as the best we can do
                if(cachedVersion < requiredVersion && (!isRelaxCacheVersionSemantics())) {
                    this.removeSessionFromManagerCache(cachedSession);
                    cachedSession = null;
                    cachedVersion = -1L;
                }
            }       
            // See if the Session is in the Store
            if(requiredVersion != -1L) {
                LoadProcessingGovernor.incrementCurrentLoads();
                session = swapIn(id, version);           
            } else {
                LoadProcessingGovernor.incrementCurrentLoads();
                session = swapIn(id);
            }               
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("findSession:swappedInSession=" + session);
            }
           
            if(session == null || session.getVersion() < cachedVersion) {
                if(_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationManagerBase>>findSession:returning cached version:" + cachedVersion);
                }           
                return cachedSession;
            }
            if(session.getVersion() < requiredVersion && (!isRelaxCacheVersionSemantics())) {           
                session = null;           
            }
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("ReplicationManagerBase>>findSession:returning:" + session);
            }        
            return (session);      
  }
    }    
   
    protected boolean isSessionOlderThan(StandardSession session, long aTime) {
        return (session.getLastAccessedTime() <= aTime);
    }
   
    public void purge(String owningInstanceName, long purgeStartTime) {
        //do not purge if we are the replica partner
        //of the owning instance
        ReplicationHealthChecker healthChecker
            = ReplicationHealthChecker.getInstance();
        String replicatedFromInstanceName
            = healthChecker.getReshapeReplicateFromInstanceName();
        if(replicatedFromInstanceName != null && replicatedFromInstanceName.equalsIgnoreCase(owningInstanceName)) {           
            return;          
        }       
        List idsToRemove = new ArrayList();
        BaseCache replicatedSessionsCache = getReplicatedSessions();
        Iterator it = replicatedSessionsCache.values();
        while(it.hasNext()) {
            ReplicationState nextState
                = (ReplicationState)it.next();
            byte[] extraParamsState
                = nextState.getContainerExtraParamsState();
            HttpSessionExtraParams extraParams = null;
            if(extraParamsState != null) {
                try {
                extraParams
                    = (HttpSessionExtraParams)ReplicationState.getObjectValue(extraParamsState);
                } catch (Exception ex) {
                    _logger.warning("unable to deserialize HttpSession extraParams for id " + nextState.getId() + ":" + ex);
                }
            }
            if(extraParams != null) {
                String nextOwningInstance = extraParams.getCurrentOwnerInstanceName();
                if(nextOwningInstance != null
                    && nextOwningInstance.equalsIgnoreCase(owningInstanceName)) {
                    idsToRemove.add((String)nextState.getId());
                }
            }
        }
        for(int i=0; i< idsToRemove.size(); i++) {
            removeFromReplicationCache((String)idsToRemove.get(i));
        }
    }      
   
    protected boolean isMonitoringEnabled() {
        return ServerConfigReader.isMonitoringEnabled();
    }
   
    public void resetMonitorStats() {
        WebModuleStatistics stats = this.getWebModuleStatistics();
        stats.resetStats();
    }    
   
    //SJSAS 6406580 START
    /** return the StorePool */
    public StorePool getStorePool() {
        return _pool;
    }
   
    /** set the StorePool */
    public void setStorePool(StorePool pool) {
        _pool = pool;
    }
    //SJSAS 6406580 END

    public ReplicationStore getSingletonStore() {
        return (ReplicationStore)this.getStore();
    }

    /** Returns a store from the pool This method intializes the store with right parameters
     * @return returns HAStorePoolElement
     */
    private HAStorePoolElement getSessionStore() {
        return getSessionStore(ReplicationUtil.isSynchronousReplicationConfigured());
    }
   
    /** Returns a store from the pool This method intializes the store with right parameters
     * @return returns HAStorePoolElement
     */
    private HAStorePoolElement getSessionStore(boolean useSingleton) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationManagerBase", "getSessionStore");
        }
        if(useSingleton) {
            return this.getSingletonStore();
        }
        if(_pool == null) {
            return null;
        }       
        HAStorePoolElement store = null;
        try {
            store = (HAStorePoolElement) _pool.take();
            store.setManager(this);
            if(_logger.isLoggable(Level.FINEST)) {
                _logger.log(Level.FINEST,
                    "ReplicationManagerBase.getSessionStore returning   " + store);
            }
            return store;
        }
        catch (Exception e) {
            if (_logger.isLoggable(Level.FINE)) {
                _logger.log(Level.FINE, "exception occurred in getSessionStore", e);
            }
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationManagerBase", "getSessionStore", store);
        }
        return store;
    }

    /**
     *  Returns (puts) a store back to the pool
     */
    private void putSessionStore(HAStorePoolElement store) {
        putSessionStore(store, ReplicationUtil.isSynchronousReplicationConfigured());
    }
   
    /**
     *  Returns (puts) a store back to the pool
     */
    private void putSessionStore(HAStorePoolElement store, boolean useSingleton) {
        if(useSingleton) {
            return;
        }
        if (store != null) {
            store.setManager(null);      
            try {
                StorePool storePool = this.getStorePool();
                if(storePool != null) {
                    storePool.put( (StorePoolElement) store);
                }
            }
            catch (InterruptedException ex1) {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.log(Level.FINE, "exception occurred in putSessionStore", ex1);
                }
            }
        }
    }   
   
    /** return the session factory */
    SessionFactory getSessionFactory() {
        return _sessionFactory;
    }
   
    /**
     * set the session factory
     * @param sessionFactory
     */
    public void setSessionFactory(SessionFactory sessionFactory) {
        _sessionFactory = sessionFactory;
    }
   
    /** should relax cache version semantics be applied */
    boolean isRelaxCacheVersionSemantics() {
        return _relaxCacheVersionSemantics;
    }
   
    /**
     * set the relaxCacheVersionSemantics
     * @param value
     */
    public void setRelaxCacheVersionSemantics(boolean value) {
        _relaxCacheVersionSemantics = value;
    }

    /** should skip backup and restore step for rolling upgrade */
    boolean isSkipRollingUpgradeBackupRestore() {
        return _skipRollingUpgradeBackupRestore;
    }

    /**
     * set the skipRollingUpgradeBackupRestore
     * @param value
     */
    public void setSkipRollingUpgradeBackupRestore(boolean value) {
        _skipRollingUpgradeBackupRestore = value;
    }

    /**
     * get the rollingUpgradeBackupDirectory
     */
    public String getRollingUpgradeBackupDirectory() {
        return _rollingUpgradeBackupDirectory;
    }

    /**
     * set the rollingUpgradeBackupDirectory
     * @param value
     */
    public void setRollingUpgradeBackupDirectory(String value) {
        _rollingUpgradeBackupDirectory = value;
    }
   
    /** return the web module statistics */
    public WebModuleStatistics getWebModuleStatistics() {
        return _statistics;
    }
   
    /**
     * set the web module statistics
     * @param statistics
     */   
    public void setWebModuleStatistics(WebModuleStatistics statistics) {
        _statistics = statistics;
    }
   
    public boolean isSessionVersioningSupported() {
        return true;
    }
   
    public String getPassedInPersistenceType() {
        return _passedInPersistenceType;
    }   
   
    public void setPassedInPersistenceType(String persistenceType) {
        _passedInPersistenceType = persistenceType;
    }
   
    public boolean isDuplicateIdsSemanticsAllowed() {
        return _duplicateIdsSemanticsAllowed;
    }   
   
    public void setDuplicateIdsSemanticsAllowed(boolean value) {
        _duplicateIdsSemanticsAllowed = value;
    }
   
    public boolean isActiveCacheReconciliationOngoing() {
        return _activeCacheReconciliationOngoing.get();
    }   
   
    public void setActiveCacheReconciliationOngoing(boolean value) {
        _activeCacheReconciliationOngoing.set(value);
    }
   
    boolean isReplicationCompressionEnabled() {
        return replicationCompressionEnabled;
    }

    ReplicaCache getReplicaCache() {
        return replicaCache;
    }

    public void setReplicaCache(ReplicaCache replicaCache) {
        this.replicaCache = replicaCache;
    }

    /**
    * Our ReplicaCache - the receiving side for replicas
    */   
    protected ReplicaCache replicaCache = null;
    protected String _passedInPersistenceType = null;
    protected boolean _duplicateIdsSemanticsAllowed = false;
    protected SessionFactory _sessionFactory = null;
    protected boolean _relaxCacheVersionSemantics = false;
    protected boolean _skipRollingUpgradeBackupRestore = false;
    protected String _rollingUpgradeBackupDirectory = null;
    protected WebModuleStatistics _statistics = new WebModuleStatistics();
    //SJSAS 6406580 START
    StorePool _pool = null;
    //SJSAS 6406580 END
    protected AtomicBoolean _activeCacheReconciliationOngoing
        = new AtomicBoolean(false);
    ConcurrentHashMap stillOwnedSessionIdsForActiveCacheReconciliation = null;
    private volatile HashMap<String, ExpatListElement> expatIdsMap =
        new HashMap<String, ExpatListElement>();
   
    private boolean replicationCompressionEnabled = false;

    public <V> V __load(String id,
                        String version,
                        JxtaBackingStoreImpl jxtaBackingStore) throws BackingStoreException {
        // TODO :: implement this.
        ReplicationStore store = getSingletonStore();
        return (V) store.__load(id, version);
    }

    private Set<String> remotelyLoadedSessionIds =
            Collections.synchronizedSet(new HashSet<String>());

    public Set<String> getRemotelyLoadedSessionIds() {
        return remotelyLoadedSessionIds;
    }

    private boolean isConverged;

    public void setIsConverged(boolean isConverged) {
        this.isConverged = isConverged;
    }

    public boolean isConverged() {
        return isConverged;
    }
}
TOP

Related Classes of com.sun.enterprise.ee.web.sessmgmt.ReplicationManagerBase

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.